{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "# First ensemble with NSL-KDD\n",
    "# Parameters\n",
    "\n",
    "#----------------------------------------------\n",
    "# 0 for not using it as base learner\n",
    "# 1 for using it as base learner\n",
    "\n",
    "use_model_ada = 1 \n",
    "use_model_dnn = 1 \n",
    "use_model_mlp = 1 \n",
    "use_model_lgbm = 1 \n",
    "use_model_rf = 1 \n",
    "use_model_svm = 1\n",
    "use_model_knn = 1 \n",
    "#----------------------------------------------\n",
    "# 0 for training the model\n",
    "# 1 for using the saved version of the model\n",
    "\n",
    "load_model_ada = 0 \n",
    "load_model_dnn = 0 \n",
    "load_model_mlp = 0 \n",
    "load_model_lgbm = 0 \n",
    "load_model_rf = 0 \n",
    "load_model_svm = 0\n",
    "load_model_knn = 0 \n",
    "#----------------------------------------------\n",
    "\n",
    "# load_model_ada = 1\n",
    "# load_model_dnn = 1 \n",
    "# load_model_mlp = 1 \n",
    "# load_model_lgbm = 1 \n",
    "# load_model_rf = 1 \n",
    "# load_model_svm = 1\n",
    "# load_model_knn = 1 \n",
    "#----------------------------------------------\n",
    "# feature_selection_bit = 0\n",
    "feature_selection_bit = 1\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Specify the name of the output text file\n",
    "if feature_selection_bit == 0:\n",
    "    output_file_name = \"ensemble_base_models_all_features_cic.txt\"\n",
    "    with open(output_file_name, \"w\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('---- ensemble_base_models_all_features', file = f)\n",
    "\n",
    "elif feature_selection_bit == 1:\n",
    "    output_file_name = \"ensemble_base_models_feature_selection_cic.txt\"\n",
    "    with open(output_file_name, \"w\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('----ensemble_base_models_feature_selection--', file = f)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!/usr/bin/env python       \n",
    "# coding: utf-8\n",
    "\n",
    "# In[1]:\n",
    "# importing required libraries\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle # saving and loading trained model\n",
    "from os import path\n",
    "\n",
    "\n",
    "# importing required libraries for normalizing data\n",
    "from sklearn import preprocessing\n",
    "from sklearn.preprocessing import (StandardScaler, OrdinalEncoder,LabelEncoder, MinMaxScaler, OneHotEncoder)\n",
    "from sklearn.preprocessing import Normalizer, MaxAbsScaler , RobustScaler, PowerTransformer\n",
    "\n",
    "# importing library for plotting\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "from sklearn import metrics\n",
    "from sklearn.metrics import accuracy_score # for calculating accuracy of model\n",
    "from sklearn.model_selection import train_test_split # for splitting the dataset for training and testing\n",
    "from sklearn.metrics import classification_report # for generating a classification report of model\n",
    "\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.metrics import f1_score\n",
    "\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "\n",
    "from keras.layers import Dense # importing dense layer\n",
    "\n",
    "from keras.layers import Input\n",
    "from keras.models import Model\n",
    "# representation of model layers\n",
    "#from keras.utils import plot_model\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import shap\n",
    "\n",
    "from sklearn.metrics import balanced_accuracy_score\n",
    "from sklearn.metrics import matthews_corrcoef\n",
    "from sklearn.metrics import roc_auc_score\n",
    "import time\n",
    "start_program = time.time()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def confusion_metrics (name_model,predictions,true_labels):\n",
    "\n",
    "    name = name_model\n",
    "    pred_label = predictions\n",
    "    y_test_01 = true_labels \n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print(name, file = f)\n",
    "\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('CONFUSION MATRIX')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "    # pred_label = label[ypred]\n",
    "\n",
    "    confusion_matrix = pd.crosstab(y_test_01, pred_label,rownames=['Actual ALERT'],colnames = ['Predicted ALERT'], dropna=False).sort_index(axis=0).sort_index(axis=1)\n",
    "    all_unique_values = sorted(set(pred_label) | set(y_test_01))\n",
    "    z = np.zeros((len(all_unique_values), len(all_unique_values)))\n",
    "    rows, cols = confusion_matrix.shape\n",
    "    z[:rows, :cols] = confusion_matrix\n",
    "    confusion_matrix  = pd.DataFrame(z, columns=all_unique_values, index=all_unique_values)\n",
    "    # confusion_matrix.to_csv('Ensemble_conf_matrix.csv')\n",
    "    # with open(output_file_name, \"a\") as f:print(confusion_matrix,file=f)\n",
    "    print(confusion_matrix)\n",
    "    with open(output_file_name, \"a\") as f: print('Confusion Matrix', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print(confusion_matrix, file = f)\n",
    "\n",
    "\n",
    "    FP = confusion_matrix.sum(axis=0) - np.diag(confusion_matrix)\n",
    "    FN = confusion_matrix.sum(axis=1) - np.diag(confusion_matrix)\n",
    "    TP = np.diag(confusion_matrix)\n",
    "    TN = confusion_matrix.values.sum() - (FP + FN + TP)\n",
    "    TP_total = sum(TP)\n",
    "    TN_total = sum(TN)\n",
    "    FP_total = sum(FP)\n",
    "    FN_total = sum(FN)\n",
    "\n",
    "    TP_total = np.array(TP_total,dtype=np.float64)\n",
    "    TN_total = np.array(TN_total,dtype=np.float64)\n",
    "    FP_total = np.array(FP_total,dtype=np.float64)\n",
    "    FN_total = np.array(FN_total,dtype=np.float64)\n",
    "\n",
    "\n",
    "\n",
    "    #----------------------------------------------------------------#----------------------------------------------------------------\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('METRICS')\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "    Acc = accuracy_score(y_test_01, pred_label)\n",
    "    Precision = precision_score(y_test_01, pred_label, average='macro')\n",
    "    Recall = recall_score(y_test_01, pred_label, average='macro')\n",
    "    F1 =  f1_score(y_test_01, pred_label, average='macro')\n",
    "    BACC = balanced_accuracy_score(y_test_01, pred_label)\n",
    "    MCC = matthews_corrcoef(y_test_01, pred_label)\n",
    "\n",
    "\n",
    "    # voting_acc_01 = Acc\n",
    "    # voting_pre_01 = Precision\n",
    "    # weighed_avg_rec_01 = Recall\n",
    "    # weighed_avg_f1_01 = F1\n",
    "    # weighed_avg_bacc_01 = BACC\n",
    "    # weighed_avg_mcc_01 = MCC\n",
    "    # with open(output_file_name, \"a\") as f:print('Accuracy total: ', Acc,file=f)\n",
    "    print('Accuracy total: ', Acc)\n",
    "    print('Precision total: ', Precision )\n",
    "    print('Recall total: ', Recall )\n",
    "    print('F1 total: ', F1 )\n",
    "    print('BACC total: ', BACC)\n",
    "    print('MCC total: ', MCC)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Accuracy total: ', Acc, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Precision total: ', Precision, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Recall total: ', Recall , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('F1 total: ', F1, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('BACC total: ', BACC , file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('MCC total: ', MCC, file = f)\n",
    "\n",
    "    return Acc, Precision, Recall, F1, BACC, MCC\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "req_cols = [' Destination Port',' Flow Duration',' Total Fwd Packets',' Total Backward Packets','Total Length of Fwd Packets',' Total Length of Bwd Packets',' Fwd Packet Length Max',' Fwd Packet Length Min',' Fwd Packet Length Mean',' Fwd Packet Length Std','Bwd Packet Length Max',' Bwd Packet Length Min',' Bwd Packet Length Mean',' Bwd Packet Length Std','Flow Bytes/s',' Flow Packets/s',' Flow IAT Mean',' Flow IAT Std',' Flow IAT Max',' Flow IAT Min','Fwd IAT Total',' Fwd IAT Mean',' Fwd IAT Std',' Fwd IAT Max',' Fwd IAT Min','Bwd IAT Total',' Bwd IAT Mean',' Bwd IAT Std',' Bwd IAT Max',' Bwd IAT Min','Fwd PSH Flags',' Bwd PSH Flags',' Fwd URG Flags',' Bwd URG Flags',' Fwd Header Length',' Bwd Header Length','Fwd Packets/s',' Bwd Packets/s',' Min Packet Length',' Max Packet Length',' Packet Length Mean',' Packet Length Std',' Packet Length Variance','FIN Flag Count',' SYN Flag Count',' RST Flag Count',' PSH Flag Count',' ACK Flag Count',' URG Flag Count',' CWE Flag Count',' ECE Flag Count',' Down/Up Ratio',' Average Packet Size',' Avg Fwd Segment Size',' Avg Bwd Segment Size',' Fwd Header Length','Fwd Avg Bytes/Bulk',' Fwd Avg Packets/Bulk',' Fwd Avg Bulk Rate',' Bwd Avg Bytes/Bulk',' Bwd Avg Packets/Bulk','Bwd Avg Bulk Rate','Subflow Fwd Packets',' Subflow Fwd Bytes',' Subflow Bwd Packets',' Subflow Bwd Bytes','Init_Win_bytes_forward',' Init_Win_bytes_backward',' act_data_pkt_fwd',' min_seg_size_forward','Active Mean',' Active Std',' Active Max',' Active Min','Idle Mean',' Idle Std',' Idle Max',' Idle Min',' Label']\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Loading Databases\n",
      "---------------------------------------------------------------------------------\n",
      "\n"
     ]
    }
   ],
   "source": [
    "\n",
    "path_str = '/home/oarreche@ads.iu.edu/HITL/cicids/cicids_db/'\n",
    "fraction = 0.333\n",
    "#---------------------------------------------------------------------\n",
    "#Load Databases from csv file\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Loading Databases')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('')\n",
    "\n",
    "\n",
    "df0 = pd.read_csv (path_str + 'Wednesday-workingHours.pcap_ISCX.csv', usecols=req_cols).sample(frac = fraction)\n",
    "\n",
    "df1 = pd.read_csv (path_str + 'Tuesday-WorkingHours.pcap_ISCX.csv', usecols=req_cols).sample(frac = fraction)\n",
    "\n",
    "\n",
    "df2 = pd.read_csv (path_str +'Thursday-WorkingHours-Morning-WebAttacks.pcap_ISCX.csv', usecols=req_cols).sample(frac = fraction)\n",
    "\n",
    "\n",
    "df3 = pd.read_csv (path_str +'Thursday-WorkingHours-Afternoon-Infilteration.pcap_ISCX.csv', usecols=req_cols).sample(frac = fraction)\n",
    "\n",
    "\n",
    "df4 = pd.read_csv (path_str +'Monday-WorkingHours.pcap_ISCX.csv', usecols=req_cols).sample(frac = fraction)\n",
    "\n",
    "\n",
    "df5 = pd.read_csv (path_str +'Friday-WorkingHours-Morning.pcap_ISCX.csv', usecols=req_cols).sample(frac = fraction)\n",
    "\n",
    "\n",
    "df6 = pd.read_csv (path_str +'Friday-WorkingHours-Afternoon-DDos.pcap_ISCX.csv', usecols=req_cols).sample(frac = fraction)\n",
    "\n",
    "\n",
    "df7 = pd.read_csv (path_str +'Friday-WorkingHours-Afternoon-PortScan.pcap_ISCX.csv', usecols=req_cols).sample(frac = fraction)\n",
    "\n",
    "\n",
    "frames = [df0, df1, df2, df3, df4, df5,df6, df7]\n",
    "\n",
    "df = pd.concat(frames,ignore_index=True)\n",
    "\n",
    "df = df.sample(frac = 1)\n",
    "#---------------------------------------------------------------------\n",
    "# Normalize database\n",
    "\n",
    "df = pd.concat(frames,ignore_index=True)\n",
    "df = df.sample(frac = fraction )\n",
    "y = df.pop(' Label')\n",
    "df = df.assign(Label = y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Reducing Normal rows\n",
      "---------------------------------------------------------------------------------\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "779010        DDoS\n",
       "831505        DDoS\n",
       "100039    DoS Hulk\n",
       "926016    PortScan\n",
       "937380    PortScan\n",
       "            ...   \n",
       "596452      BENIGN\n",
       "306497      BENIGN\n",
       "203955      BENIGN\n",
       "98549       BENIGN\n",
       "777848      BENIGN\n",
       "Name: Label, Length: 187798, dtype: object"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "frac_normal = 0.5\n",
    "\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Reducing Normal rows')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('')\n",
    "\n",
    "\n",
    "#filters\n",
    "\n",
    "filtered_normal = df[df['Label'] == 'BENIGN']\n",
    "\n",
    "#reduce\n",
    "\n",
    "reduced_normal = filtered_normal.sample(frac=frac_normal)\n",
    "\n",
    "#join\n",
    "\n",
    "df = pd.concat([df[df['Label'] != 'BENIGN'], reduced_normal])\n",
    "\n",
    "''' ---------------------------------------------------------------'''\n",
    "df_max_scaled = df.copy()\n",
    "\n",
    "\n",
    "y = df_max_scaled['Label'].replace({'DoS GoldenEye': 'Dos/Ddos', 'DoS Hulk': 'Dos/Ddos', 'DoS Slowhttptest': 'Dos/Ddos', 'DoS slowloris': 'Dos/Ddos', 'Heartbleed': 'Dos/Ddos', 'DDoS': 'Dos/Ddos','FTP-Patator': 'Brute Force', 'SSH-Patator': 'Brute Force','Web Attack - Brute Force': 'Web Attack', 'Web Attack - Sql Injection': 'Web Attack', 'Web Attack - XSS': 'Web Attack'})\n",
    "\n",
    "df_max_scaled.pop('Label')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Normalizing database\n",
      "---------------------------------------------------------------------------------\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.preprocessing import MinMaxScaler\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Normalizing database')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('')\n",
    "for col in df_max_scaled.columns:\n",
    "    t = abs(df_max_scaled[col].max())\n",
    "    df_max_scaled[col] = df_max_scaled[col]/t\n",
    "df = df_max_scaled.assign( Label = y)\n",
    "#df\n",
    "df = df.fillna(0)\n",
    "\n",
    "\n",
    "y = df.pop('Label')\n",
    "X = df\n",
    "\n",
    "# df_max_scaled = df_max_scaled.fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Counter({'BENIGN': 126100, 'Dos/Ddos': 42200, 'PortScan': 17572, 'Brute Force': 1485, 'Web Attack': 224, 'Bot': 210, 'Infiltration': 7})\n"
     ]
    }
   ],
   "source": [
    "from collections import Counter\n",
    "\n",
    "counter = Counter(y)\n",
    "print(counter)\n",
    "counter_list = list(counter.values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "# counter['Bot']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Class distribution after oversampling: Counter({'BENIGN': 126100, 'Dos/Ddos': 42200, 'PortScan': 17572, 'Brute Force': 1485, 'Bot': 1485, 'Web Attack': 1485, 'Infiltration': 1485})\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn.datasets import make_classification\n",
    "from imblearn.over_sampling import SMOTE\n",
    "from collections import Counter\n",
    "\n",
    "# Apply SMOTE for oversampling\n",
    "smote = SMOTE(sampling_strategy={'BENIGN': counter['BENIGN'], \n",
    "                                'Dos/Ddos': counter['Dos/Ddos'],\n",
    "                                'PortScan':counter['PortScan'],\n",
    "                                'Brute Force':counter['Brute Force'],\n",
    "                                'Web Attack':counter['Brute Force'],\n",
    "                                'Bot':counter['Brute Force'],\n",
    "                                'Infiltration':counter['Brute Force']}, random_state=42)\n",
    "\n",
    "# smote = SMOTE(sampling_strategy={'BENIGN': 795584, 'Dos/Ddos': 380699,'PortScan':158930,'Brute Force':13835,'Web Attack':13835,'Bot':13835,'Infiltration':13835}, random_state=42)\n",
    "X_resampled, y_resampled = smote.fit_resample(X, y)\n",
    "\n",
    "# Check the class distribution after oversampling\n",
    "print(\"Class distribution after oversampling:\", Counter(y_resampled))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = X_resampled\n",
    "y , y_label =pd.factorize(y_resampled)\n",
    "\n",
    "# y = y_resampled\n",
    "\n",
    "df = X.assign(Label = y)\n",
    "# print('train len',counter)\n",
    "\n",
    "# y = df.pop('Label')\n",
    "\n",
    "# df = df.assign(Label = y)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Destination Port</th>\n",
       "      <th>Flow Duration</th>\n",
       "      <th>Total Fwd Packets</th>\n",
       "      <th>Total Backward Packets</th>\n",
       "      <th>Total Length of Fwd Packets</th>\n",
       "      <th>Total Length of Bwd Packets</th>\n",
       "      <th>Fwd Packet Length Max</th>\n",
       "      <th>Fwd Packet Length Min</th>\n",
       "      <th>Fwd Packet Length Mean</th>\n",
       "      <th>Fwd Packet Length Std</th>\n",
       "      <th>...</th>\n",
       "      <th>min_seg_size_forward</th>\n",
       "      <th>Active Mean</th>\n",
       "      <th>Active Std</th>\n",
       "      <th>Active Max</th>\n",
       "      <th>Active Min</th>\n",
       "      <th>Idle Mean</th>\n",
       "      <th>Idle Std</th>\n",
       "      <th>Idle Max</th>\n",
       "      <th>Idle Min</th>\n",
       "      <th>Label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.001221</td>\n",
       "      <td>1.516848e-02</td>\n",
       "      <td>0.000015</td>\n",
       "      <td>0.000019</td>\n",
       "      <td>0.000015</td>\n",
       "      <td>1.997762e-05</td>\n",
       "      <td>0.000806</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.001459</td>\n",
       "      <td>0.001440</td>\n",
       "      <td>...</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.001221</td>\n",
       "      <td>1.118889e-02</td>\n",
       "      <td>0.000015</td>\n",
       "      <td>0.000022</td>\n",
       "      <td>0.000015</td>\n",
       "      <td>1.997762e-05</td>\n",
       "      <td>0.000806</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.001459</td>\n",
       "      <td>0.001440</td>\n",
       "      <td>...</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.001221</td>\n",
       "      <td>8.250400e-01</td>\n",
       "      <td>0.000040</td>\n",
       "      <td>0.000022</td>\n",
       "      <td>0.000191</td>\n",
       "      <td>1.995697e-05</td>\n",
       "      <td>0.013457</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.007028</td>\n",
       "      <td>0.016572</td>\n",
       "      <td>...</td>\n",
       "      <td>0.533333</td>\n",
       "      <td>0.000009</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000009</td>\n",
       "      <td>0.000009</td>\n",
       "      <td>0.825</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.825</td>\n",
       "      <td>0.825</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.017579</td>\n",
       "      <td>8.333335e-09</td>\n",
       "      <td>0.000005</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.032702e-08</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.050692</td>\n",
       "      <td>3.083334e-07</td>\n",
       "      <td>0.000005</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000001</td>\n",
       "      <td>1.032702e-08</td>\n",
       "      <td>0.000081</td>\n",
       "      <td>0.000872</td>\n",
       "      <td>0.000337</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191807</th>\n",
       "      <td>0.001221</td>\n",
       "      <td>4.826938e-02</td>\n",
       "      <td>0.000015</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.533333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191808</th>\n",
       "      <td>0.001221</td>\n",
       "      <td>4.228344e-02</td>\n",
       "      <td>0.000015</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.533333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191809</th>\n",
       "      <td>0.001221</td>\n",
       "      <td>4.848412e-02</td>\n",
       "      <td>0.000015</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.533333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191810</th>\n",
       "      <td>0.001221</td>\n",
       "      <td>4.883082e-02</td>\n",
       "      <td>0.000015</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.533333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191811</th>\n",
       "      <td>0.001221</td>\n",
       "      <td>4.534648e-02</td>\n",
       "      <td>0.000015</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.533333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>191812 rows × 78 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         Destination Port   Flow Duration   Total Fwd Packets  \\\n",
       "0                0.001221    1.516848e-02            0.000015   \n",
       "1                0.001221    1.118889e-02            0.000015   \n",
       "2                0.001221    8.250400e-01            0.000040   \n",
       "3                0.017579    8.333335e-09            0.000005   \n",
       "4                0.050692    3.083334e-07            0.000005   \n",
       "...                   ...             ...                 ...   \n",
       "191807           0.001221    4.826938e-02            0.000015   \n",
       "191808           0.001221    4.228344e-02            0.000015   \n",
       "191809           0.001221    4.848412e-02            0.000015   \n",
       "191810           0.001221    4.883082e-02            0.000015   \n",
       "191811           0.001221    4.534648e-02            0.000015   \n",
       "\n",
       "         Total Backward Packets  Total Length of Fwd Packets  \\\n",
       "0                      0.000019                     0.000015   \n",
       "1                      0.000022                     0.000015   \n",
       "2                      0.000022                     0.000191   \n",
       "3                      0.000004                     0.000000   \n",
       "4                      0.000004                     0.000001   \n",
       "...                         ...                          ...   \n",
       "191807                 0.000004                     0.000000   \n",
       "191808                 0.000004                     0.000000   \n",
       "191809                 0.000004                     0.000000   \n",
       "191810                 0.000004                     0.000000   \n",
       "191811                 0.000004                     0.000000   \n",
       "\n",
       "         Total Length of Bwd Packets   Fwd Packet Length Max  \\\n",
       "0                       1.997762e-05                0.000806   \n",
       "1                       1.997762e-05                0.000806   \n",
       "2                       1.995697e-05                0.013457   \n",
       "3                       1.032702e-08                0.000000   \n",
       "4                       1.032702e-08                0.000081   \n",
       "...                              ...                     ...   \n",
       "191807                  0.000000e+00                0.000000   \n",
       "191808                  0.000000e+00                0.000000   \n",
       "191809                  0.000000e+00                0.000000   \n",
       "191810                  0.000000e+00                0.000000   \n",
       "191811                  0.000000e+00                0.000000   \n",
       "\n",
       "         Fwd Packet Length Min   Fwd Packet Length Mean  \\\n",
       "0                     0.000000                 0.001459   \n",
       "1                     0.000000                 0.001459   \n",
       "2                     0.000000                 0.007028   \n",
       "3                     0.000000                 0.000000   \n",
       "4                     0.000872                 0.000337   \n",
       "...                        ...                      ...   \n",
       "191807                0.000000                 0.000000   \n",
       "191808                0.000000                 0.000000   \n",
       "191809                0.000000                 0.000000   \n",
       "191810                0.000000                 0.000000   \n",
       "191811                0.000000                 0.000000   \n",
       "\n",
       "         Fwd Packet Length Std  ...   min_seg_size_forward  Active Mean  \\\n",
       "0                     0.001440  ...               0.333333     0.000000   \n",
       "1                     0.001440  ...               0.333333     0.000000   \n",
       "2                     0.016572  ...               0.533333     0.000009   \n",
       "3                     0.000000  ...               0.666667     0.000000   \n",
       "4                     0.000000  ...               0.400000     0.000000   \n",
       "...                        ...  ...                    ...          ...   \n",
       "191807                0.000000  ...               0.533333     0.000000   \n",
       "191808                0.000000  ...               0.533333     0.000000   \n",
       "191809                0.000000  ...               0.533333     0.000000   \n",
       "191810                0.000000  ...               0.533333     0.000000   \n",
       "191811                0.000000  ...               0.533333     0.000000   \n",
       "\n",
       "         Active Std   Active Max   Active Min  Idle Mean   Idle Std  \\\n",
       "0               0.0     0.000000     0.000000      0.000        0.0   \n",
       "1               0.0     0.000000     0.000000      0.000        0.0   \n",
       "2               0.0     0.000009     0.000009      0.825        0.0   \n",
       "3               0.0     0.000000     0.000000      0.000        0.0   \n",
       "4               0.0     0.000000     0.000000      0.000        0.0   \n",
       "...             ...          ...          ...        ...        ...   \n",
       "191807          0.0     0.000000     0.000000      0.000        0.0   \n",
       "191808          0.0     0.000000     0.000000      0.000        0.0   \n",
       "191809          0.0     0.000000     0.000000      0.000        0.0   \n",
       "191810          0.0     0.000000     0.000000      0.000        0.0   \n",
       "191811          0.0     0.000000     0.000000      0.000        0.0   \n",
       "\n",
       "         Idle Max   Idle Min  Label  \n",
       "0           0.000      0.000      0  \n",
       "1           0.000      0.000      0  \n",
       "2           0.825      0.825      0  \n",
       "3           0.000      0.000      1  \n",
       "4           0.000      0.000      1  \n",
       "...           ...        ...    ...  \n",
       "191807      0.000      0.000      4  \n",
       "191808      0.000      0.000      4  \n",
       "191809      0.000      0.000      4  \n",
       "191810      0.000      0.000      4  \n",
       "191811      0.000      0.000      4  \n",
       "\n",
       "[191812 rows x 78 columns]"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "# single_class_train = np.argmax(y_train_multi, axis=1)\n",
    "# single_class_test = np.argmax(y_test_multi, axis=1)\n",
    "\n",
    "\n",
    "# df1 = X_train_multi.assign(Label = single_class_train)\n",
    "# df2 =  X_test_multi.assign(Label = single_class_test)\n",
    "\n",
    "# frames = [df1,  df2]\n",
    "\n",
    "# df = pd.concat(frames,ignore_index=True)\n",
    "# df_fs = df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "# y = df.pop('Label')\n",
    "# X = df\n",
    "# df = X.assign(Label = y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "# y = df_fs.pop('Label')\n",
    "# X = df_fs\n",
    "# df_fs = X.assign(Label = y)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature Selection Methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from sklearn.tree import DecisionTreeClassifier\n",
    "# from sklearn.feature_selection import mutual_info_classif\n",
    "\n",
    "# # Split the dataset into training and testing sets\n",
    "# X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
    "\n",
    "# # Train a decision tree classifier\n",
    "# clf = DecisionTreeClassifier(random_state=42)\n",
    "# clf.fit(X_train, y_train)\n",
    "\n",
    "# # Compute information gain using mutual information\n",
    "# info_gain = mutual_info_classif(X_train, y_train)\n",
    "\n",
    "# # Display information gain for each feature\n",
    "# for feature, gain in zip(X_train.columns, info_gain):\n",
    "#     print(f'Information Gain for {feature}: {gain}')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "if feature_selection_bit == 1 and 0==1:\n",
    "\n",
    "    from sklearn.feature_selection import mutual_info_classif\n",
    "    %matplotlib inline\n",
    "\n",
    "    # Compute information gain using mutual information\n",
    "    importances = mutual_info_classif(X, y)\n",
    "\n",
    "    feat_importances = pd.Series(importances, df.columns[0:len(df.columns)])\n",
    "    # feat_importances.plot(kind='barh', color = 'teal')\n",
    "        \n",
    "    feat_importances_sorted = feat_importances.sort_values( ascending=False)\n",
    "\n",
    "    # Print or use the sorted DataFrame\n",
    "    print(feat_importances_sorted)\n",
    "    # feat_importances_sorted.plot(kind='barh', color = 'teal')\n",
    "    # feat_importances_sorted\n",
    "    top_features = feat_importances_sorted.nlargest(10)\n",
    "    top_feature_names = top_features.index.tolist()\n",
    "\n",
    "    print(\"Top 10 feature names:\")\n",
    "    print(top_feature_names)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# feat_importances_sorted = feat_importances.sort_values( ascending=False)\n",
    "\n",
    "# # Print or use the sorted DataFrame\n",
    "# print(feat_importances_sorted)\n",
    "# # feat_importances_sorted.plot(kind='barh', color = 'teal')\n",
    "# # feat_importances_sorted\n",
    "# top_features = feat_importances_sorted.nlargest(10)\n",
    "# top_feature_names = top_features.index.tolist()\n",
    "\n",
    "# print(\"Top 10 feature names:\")\n",
    "# print(top_feature_names)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from skfeature.function.similarity_based import fisher_score\n",
    "# import matplotlib.pyplot as plt\n",
    "# %matplotlib inline \n",
    "\n",
    "# ranks = fisher_score.fisher_score(X,y)\n",
    "\n",
    "# feat_importances = pd.Series(ranks, dataframe.columns[0:len(dataframe.columns)-1])\n",
    "# feat_importances.plot(kind = 'barh',color = 'teal')\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "# stop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "if feature_selection_bit == 1:\n",
    "    # USE XAI from last work\n",
    "    feature_selection = [ \n",
    "                    ' Destination Port',\n",
    "                    ' Init_Win_bytes_backward',\n",
    "                    ' Packet Length Std',\n",
    "                    ' Bwd Packet Length Mean', \n",
    "                    ' Total Length of Bwd Packets', \n",
    "                    ' Packet Length Mean',\n",
    "                    ' Subflow Bwd Bytes',\n",
    "                    ' Packet Length Variance', \n",
    "                    'Label']\n",
    "\n",
    "\n",
    "    # Use information gain\n",
    "    # feature_selection = top_feature_names\n",
    "    \n",
    "\n",
    "    df_og = df\n",
    "    df = df[feature_selection]\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# # y = df.pop('Label')\n",
    "# # X = df\n",
    "\n",
    "# y1, y2 = pd.factorize(y)\n",
    "\n",
    "# y_0 = pd.DataFrame(y1)\n",
    "# y_1 = pd.DataFrame(y1)\n",
    "# y_2 = pd.DataFrame(y1)\n",
    "# y_3 = pd.DataFrame(y1)\n",
    "# y_4 = pd.DataFrame(y1)\n",
    "\n",
    "\n",
    "# # y_0 = y_0.replace(0, 0)\n",
    "# # y_0 = y_0.replace(1, 1)\n",
    "# y_0 = y_0.replace(2, 1)\n",
    "# y_0 = y_0.replace(3, 1)\n",
    "# y_0 = y_0.replace(4, 1)\n",
    "\n",
    "\n",
    "# y_1 = y_1.replace(1, 999)\n",
    "# y_1 = y_1.replace(0, 1)\n",
    "# # y_1 = y_1.replace(1, 0)\n",
    "# y_1 = y_1.replace(2, 1)\n",
    "# y_1 = y_1.replace(3, 1)\n",
    "# y_1 = y_1.replace(4, 1)\n",
    "# y_1 = y_1.replace(999, 1)\n",
    "\n",
    "\n",
    "# y_2 = y_2.replace(0, 1)\n",
    "# y_2 = y_2.replace(1, 1)\n",
    "# y_2 = y_2.replace(2, 0)\n",
    "# y_2 = y_2.replace(3, 1)\n",
    "# y_2 = y_2.replace(4, 1)\n",
    "\n",
    "\n",
    "# y_3 = y_3.replace(0, 1)\n",
    "# # y_3 = y_3.replace(1, 1)\n",
    "# y_3 = y_3.replace(2, 1)\n",
    "# y_3 = y_3.replace(3, 0)\n",
    "# y_3 = y_3.replace(4, 1)\n",
    "\n",
    "\n",
    "# y_4 = y_4.replace(0, 1)\n",
    "# # y_4 = y_4.replace(1, 1)\n",
    "# y_4 = y_4.replace(2, 1)\n",
    "# y_4 = y_4.replace(3, 1)\n",
    "# y_4 = y_4.replace(4, 0)\n",
    "\n",
    "\n",
    "\n",
    "# df = df.assign(Label = y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "#Divide the dataset between level 00 and level 01\n",
    "import sklearn\n",
    "from sklearn.model_selection import train_test_split\n",
    "split = 0.7 \n",
    "\n",
    "X_train,X_test, y_train, y_test = sklearn.model_selection.train_test_split(X, y, train_size=split)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Counter({6: 126100, 0: 42200, 1: 17572, 2: 1485, 3: 1485, 4: 1485, 5: 1485})\n"
     ]
    }
   ],
   "source": [
    "from collections import Counter\n",
    "\n",
    "label_counts2 = Counter(y)\n",
    "print(label_counts2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Base learner Split\n",
    "# split = 0.7\n",
    "\n",
    "# X_train,X_test, y_train, y_test = sklearn.model_selection.train_test_split(X_00, y_00, train_size=split)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Destination Port</th>\n",
       "      <th>Flow Duration</th>\n",
       "      <th>Total Fwd Packets</th>\n",
       "      <th>Total Backward Packets</th>\n",
       "      <th>Total Length of Fwd Packets</th>\n",
       "      <th>Total Length of Bwd Packets</th>\n",
       "      <th>Fwd Packet Length Max</th>\n",
       "      <th>Fwd Packet Length Min</th>\n",
       "      <th>Fwd Packet Length Mean</th>\n",
       "      <th>Fwd Packet Length Std</th>\n",
       "      <th>...</th>\n",
       "      <th>act_data_pkt_fwd</th>\n",
       "      <th>min_seg_size_forward</th>\n",
       "      <th>Active Mean</th>\n",
       "      <th>Active Std</th>\n",
       "      <th>Active Max</th>\n",
       "      <th>Active Min</th>\n",
       "      <th>Idle Mean</th>\n",
       "      <th>Idle Std</th>\n",
       "      <th>Idle Max</th>\n",
       "      <th>Idle Min</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>174687</th>\n",
       "      <td>0.000809</td>\n",
       "      <td>1.556117e-03</td>\n",
       "      <td>0.000005</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000026</td>\n",
       "      <td>2.168675e-07</td>\n",
       "      <td>0.001853</td>\n",
       "      <td>0.020061</td>\n",
       "      <td>0.007743</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.533333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143676</th>\n",
       "      <td>0.056292</td>\n",
       "      <td>7.500002e-08</td>\n",
       "      <td>0.000010</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000081</td>\n",
       "      <td>0.000872</td>\n",
       "      <td>0.000337</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000005</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143854</th>\n",
       "      <td>0.680710</td>\n",
       "      <td>7.083335e-07</td>\n",
       "      <td>0.000005</td>\n",
       "      <td>0.000004</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.533333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>160483</th>\n",
       "      <td>0.000809</td>\n",
       "      <td>2.058334e-06</td>\n",
       "      <td>0.000010</td>\n",
       "      <td>0.000007</td>\n",
       "      <td>0.000039</td>\n",
       "      <td>7.263339e-07</td>\n",
       "      <td>0.001370</td>\n",
       "      <td>0.014828</td>\n",
       "      <td>0.005723</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000005</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7009</th>\n",
       "      <td>0.001221</td>\n",
       "      <td>3.344897e-02</td>\n",
       "      <td>0.000025</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000017</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000242</td>\n",
       "      <td>0.002617</td>\n",
       "      <td>0.001010</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000021</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149754</th>\n",
       "      <td>0.000809</td>\n",
       "      <td>5.110251e-04</td>\n",
       "      <td>0.000010</td>\n",
       "      <td>0.000007</td>\n",
       "      <td>0.000041</td>\n",
       "      <td>2.994836e-07</td>\n",
       "      <td>0.001450</td>\n",
       "      <td>0.015700</td>\n",
       "      <td>0.006060</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000005</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50744</th>\n",
       "      <td>0.001221</td>\n",
       "      <td>7.114358e-01</td>\n",
       "      <td>0.000035</td>\n",
       "      <td>0.000022</td>\n",
       "      <td>0.000175</td>\n",
       "      <td>1.995697e-05</td>\n",
       "      <td>0.012369</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.007382</td>\n",
       "      <td>0.016284</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000005</td>\n",
       "      <td>0.533333</td>\n",
       "      <td>0.000003</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000003</td>\n",
       "      <td>0.000003</td>\n",
       "      <td>0.710000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.710000</td>\n",
       "      <td>0.710000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143975</th>\n",
       "      <td>0.006760</td>\n",
       "      <td>6.700205e-01</td>\n",
       "      <td>0.000096</td>\n",
       "      <td>0.000089</td>\n",
       "      <td>0.000600</td>\n",
       "      <td>3.208778e-05</td>\n",
       "      <td>0.015673</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.009311</td>\n",
       "      <td>0.016983</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000096</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.000853</td>\n",
       "      <td>0.004754</td>\n",
       "      <td>0.005279</td>\n",
       "      <td>0.000220</td>\n",
       "      <td>0.083006</td>\n",
       "      <td>0.000932</td>\n",
       "      <td>0.083319</td>\n",
       "      <td>0.081745</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>156277</th>\n",
       "      <td>0.000809</td>\n",
       "      <td>9.111086e-04</td>\n",
       "      <td>0.000010</td>\n",
       "      <td>0.000007</td>\n",
       "      <td>0.000042</td>\n",
       "      <td>6.161790e-07</td>\n",
       "      <td>0.001491</td>\n",
       "      <td>0.016136</td>\n",
       "      <td>0.006228</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000005</td>\n",
       "      <td>0.533333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>157569</th>\n",
       "      <td>0.000809</td>\n",
       "      <td>1.658334e-06</td>\n",
       "      <td>0.000010</td>\n",
       "      <td>0.000007</td>\n",
       "      <td>0.000035</td>\n",
       "      <td>3.855422e-07</td>\n",
       "      <td>0.001249</td>\n",
       "      <td>0.013519</td>\n",
       "      <td>0.005218</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000005</td>\n",
       "      <td>0.533333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>134268 rows × 77 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         Destination Port   Flow Duration   Total Fwd Packets  \\\n",
       "174687           0.000809    1.556117e-03            0.000005   \n",
       "143676           0.056292    7.500002e-08            0.000010   \n",
       "143854           0.680710    7.083335e-07            0.000005   \n",
       "160483           0.000809    2.058334e-06            0.000010   \n",
       "7009             0.001221    3.344897e-02            0.000025   \n",
       "...                   ...             ...                 ...   \n",
       "149754           0.000809    5.110251e-04            0.000010   \n",
       "50744            0.001221    7.114358e-01            0.000035   \n",
       "143975           0.006760    6.700205e-01            0.000096   \n",
       "156277           0.000809    9.111086e-04            0.000010   \n",
       "157569           0.000809    1.658334e-06            0.000010   \n",
       "\n",
       "         Total Backward Packets  Total Length of Fwd Packets  \\\n",
       "174687                 0.000004                     0.000026   \n",
       "143676                 0.000000                     0.000002   \n",
       "143854                 0.000004                     0.000000   \n",
       "160483                 0.000007                     0.000039   \n",
       "7009                   0.000000                     0.000017   \n",
       "...                         ...                          ...   \n",
       "149754                 0.000007                     0.000041   \n",
       "50744                  0.000022                     0.000175   \n",
       "143975                 0.000089                     0.000600   \n",
       "156277                 0.000007                     0.000042   \n",
       "157569                 0.000007                     0.000035   \n",
       "\n",
       "         Total Length of Bwd Packets   Fwd Packet Length Max  \\\n",
       "174687                  2.168675e-07                0.001853   \n",
       "143676                  0.000000e+00                0.000081   \n",
       "143854                  0.000000e+00                0.000000   \n",
       "160483                  7.263339e-07                0.001370   \n",
       "7009                    0.000000e+00                0.000242   \n",
       "...                              ...                     ...   \n",
       "149754                  2.994836e-07                0.001450   \n",
       "50744                   1.995697e-05                0.012369   \n",
       "143975                  3.208778e-05                0.015673   \n",
       "156277                  6.161790e-07                0.001491   \n",
       "157569                  3.855422e-07                0.001249   \n",
       "\n",
       "         Fwd Packet Length Min   Fwd Packet Length Mean  \\\n",
       "174687                0.020061                 0.007743   \n",
       "143676                0.000872                 0.000337   \n",
       "143854                0.000000                 0.000000   \n",
       "160483                0.014828                 0.005723   \n",
       "7009                  0.002617                 0.001010   \n",
       "...                        ...                      ...   \n",
       "149754                0.015700                 0.006060   \n",
       "50744                 0.000000                 0.007382   \n",
       "143975                0.000000                 0.009311   \n",
       "156277                0.016136                 0.006228   \n",
       "157569                0.013519                 0.005218   \n",
       "\n",
       "         Fwd Packet Length Std  ...   act_data_pkt_fwd   min_seg_size_forward  \\\n",
       "174687                0.000000  ...           0.000000               0.533333   \n",
       "143676                0.000000  ...           0.000005               0.400000   \n",
       "143854                0.000000  ...           0.000000               0.533333   \n",
       "160483                0.000000  ...           0.000005               0.333333   \n",
       "7009                  0.000000  ...           0.000021               0.333333   \n",
       "...                        ...  ...                ...                    ...   \n",
       "149754                0.000000  ...           0.000005               0.333333   \n",
       "50744                 0.016284  ...           0.000005               0.533333   \n",
       "143975                0.016983  ...           0.000096               0.333333   \n",
       "156277                0.000000  ...           0.000005               0.533333   \n",
       "157569                0.000000  ...           0.000005               0.533333   \n",
       "\n",
       "        Active Mean   Active Std   Active Max   Active Min  Idle Mean  \\\n",
       "174687     0.000000     0.000000     0.000000     0.000000   0.000000   \n",
       "143676     0.000000     0.000000     0.000000     0.000000   0.000000   \n",
       "143854     0.000000     0.000000     0.000000     0.000000   0.000000   \n",
       "160483     0.000000     0.000000     0.000000     0.000000   0.000000   \n",
       "7009       0.000000     0.000000     0.000000     0.000000   0.000000   \n",
       "...             ...          ...          ...          ...        ...   \n",
       "149754     0.000000     0.000000     0.000000     0.000000   0.000000   \n",
       "50744      0.000003     0.000000     0.000003     0.000003   0.710000   \n",
       "143975     0.000853     0.004754     0.005279     0.000220   0.083006   \n",
       "156277     0.000000     0.000000     0.000000     0.000000   0.000000   \n",
       "157569     0.000000     0.000000     0.000000     0.000000   0.000000   \n",
       "\n",
       "         Idle Std   Idle Max   Idle Min  \n",
       "174687   0.000000   0.000000   0.000000  \n",
       "143676   0.000000   0.000000   0.000000  \n",
       "143854   0.000000   0.000000   0.000000  \n",
       "160483   0.000000   0.000000   0.000000  \n",
       "7009     0.000000   0.000000   0.000000  \n",
       "...           ...        ...        ...  \n",
       "149754   0.000000   0.000000   0.000000  \n",
       "50744    0.000000   0.710000   0.710000  \n",
       "143975   0.000932   0.083319   0.081745  \n",
       "156277   0.000000   0.000000   0.000000  \n",
       "157569   0.000000   0.000000   0.000000  \n",
       "\n",
       "[134268 rows x 77 columns]"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([6, 6, 6, ..., 6, 6, 6])"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LEVEL 0 - Weak models - Base Learner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining RF Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Defining ADA Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Defining LGBM Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Defining KNN Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Defining SVM Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Defining MLP Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Defining DNN Model\n",
      "---------------------------------------------------------------------------------\n",
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_6 (Dense)              (None, 3)                 234       \n",
      "_________________________________________________________________\n",
      "dropout_5 (Dropout)          (None, 3)                 0         \n",
      "_________________________________________________________________\n",
      "dense_7 (Dense)              (None, 3)                 12        \n",
      "_________________________________________________________________\n",
      "dropout_6 (Dropout)          (None, 3)                 0         \n",
      "_________________________________________________________________\n",
      "dense_8 (Dense)              (None, 3)                 12        \n",
      "_________________________________________________________________\n",
      "dropout_7 (Dropout)          (None, 3)                 0         \n",
      "_________________________________________________________________\n",
      "dense_9 (Dense)              (None, 3)                 12        \n",
      "_________________________________________________________________\n",
      "dropout_8 (Dropout)          (None, 3)                 0         \n",
      "_________________________________________________________________\n",
      "dense_10 (Dense)             (None, 3)                 12        \n",
      "_________________________________________________________________\n",
      "dropout_9 (Dropout)          (None, 3)                 0         \n",
      "_________________________________________________________________\n",
      "dense_11 (Dense)             (None, 7)                 28        \n",
      "=================================================================\n",
      "Total params: 310\n",
      "Trainable params: 310\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "with open(output_file_name, \"a\") as f: print('------------START of WEAK LEARNERS (BASE MODELS) - STACK 00 -----------------', file = f)\n",
    "\n",
    "#Defining Basemodels\n",
    "\n",
    "\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining RF Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "#Random Forest\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.multioutput import MultiOutputClassifier\n",
    "rf = RandomForestClassifier(max_depth = 5,  n_estimators = 10, min_samples_split = 2, n_jobs = -1)\n",
    "#------------------------------------------------------------------------------\n",
    "\n",
    "\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining ADA Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "#ADA\n",
    "from sklearn.multioutput import MultiOutputClassifier\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "import time\n",
    "abc = AdaBoostClassifier(n_estimators=50, learning_rate=1.0)\n",
    "\n",
    "\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining LGBM Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "#LGBM\n",
    "from lightgbm import LGBMClassifier\n",
    "lgbm = LGBMClassifier()\n",
    "\n",
    "\n",
    "\n",
    "#KNN\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining KNN Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "knn_clf=KNeighborsClassifier(n_neighbors = 5)\n",
    "\n",
    "\n",
    "#SVM\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining SVM Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "\n",
    "from sklearn.multioutput import MultiOutputClassifier\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "\n",
    "# Instantiate the SGDClassifier with additional hyperparameters\n",
    "clf = SGDClassifier(\n",
    "    loss='hinge',           # hinge loss for linear SVM\n",
    "    penalty='l2',           # L2 regularization to prevent overfitting\n",
    "    alpha=1e-4,             # Learning rate (small value for fine-grained updates)\n",
    "    max_iter=1000,          # Number of passes over the training data\n",
    "    random_state=42,        # Seed for reproducible results\n",
    "    learning_rate='optimal' # Automatically adjusts the learning rate based on the training data\n",
    ")\n",
    "\n",
    "\n",
    "#MLP\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining MLP Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "\n",
    "\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.multioutput import MultiOutputClassifier\n",
    "import time\n",
    "\n",
    "# create MLPClassifier instance\n",
    "mlp = MLPClassifier(hidden_layer_sizes=(100,), max_iter=200, random_state=1)\n",
    "\n",
    "\n",
    "#DNN\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining DNN Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense\n",
    "\n",
    "# #Model Parameters\n",
    "# dropout_rate = 0.01\n",
    "# nodes = 70\n",
    "# out_layer = 5\n",
    "# optimizer='adam'\n",
    "# loss='sparse_categorical_crossentropy'\n",
    "# epochs=1\n",
    "# batch_size=2*256\n",
    "\n",
    "#Model Parameters\n",
    "dropout_rate = 0.2\n",
    "nodes = 3\n",
    "out_layer = 7\n",
    "optimizer='adam'\n",
    "loss='sparse_categorical_crossentropy'\n",
    "epochs=100\n",
    "batch_size=128\n",
    "\n",
    "\n",
    "num_columns = X_train.shape[1]\n",
    "\n",
    "dnn = tf.keras.Sequential()\n",
    "\n",
    "# Input layer\n",
    "dnn.add(tf.keras.Input(shape=(num_columns,)))\n",
    "\n",
    "# Dense layers with dropout\n",
    "dnn.add(tf.keras.layers.Dense(nodes))\n",
    "dnn.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "dnn.add(tf.keras.layers.Dense(nodes))\n",
    "dnn.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "dnn.add(tf.keras.layers.Dense(nodes))\n",
    "dnn.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "dnn.add(tf.keras.layers.Dense(nodes))\n",
    "dnn.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "dnn.add(tf.keras.layers.Dense(nodes))\n",
    "dnn.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "# Output layer\n",
    "dnn.add(tf.keras.layers.Dense(out_layer, activation='softmax'))\n",
    "\n",
    "dnn.compile(optimizer=optimizer, loss=loss,metrics=['accuracy'])\n",
    "\n",
    "dnn.summary()\n",
    "\n",
    "\n",
    "\n",
    "# dnn = Sequential()\n",
    "# dnn.add(Dense(128, input_dim=X_train.shape[1], activation='relu'))  # Input layer\n",
    "# dnn.add(Dense(64, activation='relu'))  # Hidden layer\n",
    "# dnn.add(Dense(5))  # Output layer\n",
    "\n",
    "# dnn.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "# # summary of model layers\n",
    "# dnn.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "# #SVM\n",
    "# # Wrap SGDClassifier with MultiOutputClassifier\n",
    "# multi_target_clf = MultiOutputClassifier(clf)\n",
    "\n",
    "# # Fit the model on the training data\n",
    "# multi_target_clf.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions on the test data\n",
    "# y_pred = clf.predict(X_test)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Training Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Training ADA\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Training RF\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Training SVM\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Training KNN\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Training LGBM\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Training MLP\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Training DNN\n",
      "---------------------------------------------------------------------------------\n",
      "Epoch 1/100\n",
      "840/840 [==============================] - 3s 3ms/step - loss: 1.0330 - accuracy: 0.6974 - val_loss: 0.5290 - val_accuracy: 0.8364\n",
      "Epoch 2/100\n",
      "840/840 [==============================] - 3s 3ms/step - loss: 0.6464 - accuracy: 0.7954 - val_loss: 0.4794 - val_accuracy: 0.8438\n",
      "Epoch 3/100\n",
      "840/840 [==============================] - 3s 3ms/step - loss: 0.5879 - accuracy: 0.8208 - val_loss: 0.4646 - val_accuracy: 0.8384\n",
      "Epoch 4/100\n",
      "840/840 [==============================] - 3s 3ms/step - loss: 0.5618 - accuracy: 0.8211 - val_loss: 0.4572 - val_accuracy: 0.8462\n",
      "Epoch 5/100\n",
      "840/840 [==============================] - 3s 3ms/step - loss: 0.5506 - accuracy: 0.8223 - val_loss: 0.4540 - val_accuracy: 0.8438\n",
      "Epoch 6/100\n",
      "840/840 [==============================] - 3s 3ms/step - loss: 0.5372 - accuracy: 0.8217 - val_loss: 0.4516 - val_accuracy: 0.8459\n",
      "Epoch 7/100\n",
      "840/840 [==============================] - 3s 3ms/step - loss: 0.5296 - accuracy: 0.8233 - val_loss: 0.4509 - val_accuracy: 0.8370\n",
      "Epoch 8/100\n",
      "840/840 [==============================] - 3s 3ms/step - loss: 0.5206 - accuracy: 0.8244 - val_loss: 0.4470 - val_accuracy: 0.8408\n",
      "Epoch 9/100\n",
      "840/840 [==============================] - 3s 3ms/step - loss: 0.5109 - accuracy: 0.8250 - val_loss: 0.4518 - val_accuracy: 0.8461\n",
      "Epoch 10/100\n",
      "840/840 [==============================] - 3s 3ms/step - loss: 0.5066 - accuracy: 0.8261 - val_loss: 0.4487 - val_accuracy: 0.8464\n",
      "Epoch 11/100\n",
      "840/840 [==============================] - 3s 3ms/step - loss: 0.4946 - accuracy: 0.8273 - val_loss: 0.4463 - val_accuracy: 0.8416\n",
      "Epoch 12/100\n",
      "840/840 [==============================] - 3s 3ms/step - loss: 0.4949 - accuracy: 0.8278 - val_loss: 0.4504 - val_accuracy: 0.8458\n",
      "Epoch 13/100\n",
      "840/840 [==============================] - 3s 3ms/step - loss: 0.4914 - accuracy: 0.8288 - val_loss: 0.4492 - val_accuracy: 0.8448\n",
      "Epoch 14/100\n",
      "840/840 [==============================] - 3s 3ms/step - loss: 0.4929 - accuracy: 0.8299 - val_loss: 0.4504 - val_accuracy: 0.8428\n",
      "Epoch 15/100\n",
      "840/840 [==============================] - 3s 3ms/step - loss: 0.4954 - accuracy: 0.8304 - val_loss: 0.4476 - val_accuracy: 0.8480\n",
      "Epoch 16/100\n",
      "840/840 [==============================] - 3s 3ms/step - loss: 0.4955 - accuracy: 0.8306 - val_loss: 0.4473 - val_accuracy: 0.8433\n",
      "Epoch 17/100\n",
      "840/840 [==============================] - 3s 3ms/step - loss: 0.4888 - accuracy: 0.8307 - val_loss: 0.4591 - val_accuracy: 0.8452\n",
      "Epoch 18/100\n",
      "840/840 [==============================] - 3s 3ms/step - loss: 0.4885 - accuracy: 0.8317 - val_loss: 0.4530 - val_accuracy: 0.8470\n",
      "Epoch 19/100\n",
      "840/840 [==============================] - 3s 3ms/step - loss: 0.4951 - accuracy: 0.8304 - val_loss: 0.4464 - val_accuracy: 0.8449\n",
      "Epoch 20/100\n",
      "840/840 [==============================] - 3s 3ms/step - loss: 0.4856 - accuracy: 0.8318 - val_loss: 0.4495 - val_accuracy: 0.8506\n",
      "Epoch 21/100\n",
      "840/840 [==============================] - 3s 3ms/step - loss: 0.4843 - accuracy: 0.8313 - val_loss: 0.4487 - val_accuracy: 0.8468\n",
      "Epoch 22/100\n",
      "840/840 [==============================] - 3s 3ms/step - loss: 0.4856 - accuracy: 0.8313 - val_loss: 0.4510 - val_accuracy: 0.8470\n",
      "Epoch 23/100\n",
      "840/840 [==============================] - 3s 3ms/step - loss: 0.4925 - accuracy: 0.8324 - val_loss: 0.4468 - val_accuracy: 0.8464\n",
      "Epoch 24/100\n",
      "840/840 [==============================] - 3s 3ms/step - loss: 0.8682 - accuracy: 0.8328 - val_loss: 0.4573 - val_accuracy: 0.8404\n",
      "Epoch 25/100\n",
      "840/840 [==============================] - 3s 3ms/step - loss: 0.4918 - accuracy: 0.8322 - val_loss: 0.4490 - val_accuracy: 0.8470\n",
      "Epoch 26/100\n",
      "840/840 [==============================] - 3s 3ms/step - loss: 0.4878 - accuracy: 0.8324 - val_loss: 0.4543 - val_accuracy: 0.8526\n",
      "Epoch 27/100\n",
      "840/840 [==============================] - 3s 3ms/step - loss: 0.4978 - accuracy: 0.8326 - val_loss: 0.4498 - val_accuracy: 0.8551\n",
      "Epoch 28/100\n",
      "840/840 [==============================] - 3s 3ms/step - loss: 0.4894 - accuracy: 0.8332 - val_loss: 0.4529 - val_accuracy: 0.8487\n",
      "Epoch 29/100\n",
      "840/840 [==============================] - 3s 3ms/step - loss: 0.4932 - accuracy: 0.8321 - val_loss: 0.4529 - val_accuracy: 0.8421\n",
      "Epoch 30/100\n",
      "840/840 [==============================] - 3s 3ms/step - loss: 0.4922 - accuracy: 0.8325 - val_loss: 0.4505 - val_accuracy: 0.8444\n",
      "Epoch 31/100\n",
      "840/840 [==============================] - 3s 3ms/step - loss: 0.4866 - accuracy: 0.8331 - val_loss: 0.4486 - val_accuracy: 0.8508\n",
      "Epoch 32/100\n",
      "840/840 [==============================] - 3s 3ms/step - loss: 0.4848 - accuracy: 0.8340 - val_loss: 0.4495 - val_accuracy: 0.8492\n",
      "Epoch 33/100\n",
      "840/840 [==============================] - 3s 3ms/step - loss: 0.4856 - accuracy: 0.8339 - val_loss: 0.4521 - val_accuracy: 0.8429\n",
      "Epoch 34/100\n",
      "840/840 [==============================] - 3s 3ms/step - loss: 0.4915 - accuracy: 0.8334 - val_loss: 0.4558 - val_accuracy: 0.8469\n",
      "Epoch 35/100\n",
      "840/840 [==============================] - 3s 3ms/step - loss: 0.4853 - accuracy: 0.8325 - val_loss: 0.4498 - val_accuracy: 0.8510\n",
      "Epoch 36/100\n",
      "840/840 [==============================] - 3s 3ms/step - loss: 0.4908 - accuracy: 0.8334 - val_loss: 0.4454 - val_accuracy: 0.8540\n",
      "Epoch 37/100\n",
      "840/840 [==============================] - 3s 3ms/step - loss: 0.4883 - accuracy: 0.8329 - val_loss: 0.4468 - val_accuracy: 0.8491\n"
     ]
    }
   ],
   "source": [
    "#Training Basemodels\n",
    "import joblib\n",
    "from sklearn.model_selection import StratifiedKFold, cross_val_score\n",
    "n_splits = 5  # You can adjust the number of folds as needed\n",
    "\n",
    "\n",
    "\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Training Model')\n",
    "with open(output_file_name, \"a\") as f: print('Training weak models - level 0', file = f)\n",
    "\n",
    "print('---------------------------------------------------------------------------------')\n",
    "\n",
    "if use_model_ada == 1 and load_model_ada == 0:\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training ADA')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training ADA', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #ADA\n",
    "\n",
    "\n",
    "    start = time.time()\n",
    "    ada = abc.fit(X_train, y_train)\n",
    "    end = time.time()\n",
    "\n",
    "    # # Create the StratifiedKFold object\n",
    "    # stratified_kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "    # # Perform cross-validation\n",
    "    # cv_scores = cross_val_score(ada, X_train, y_train, cv=stratified_kfold, scoring='accuracy')\n",
    "    # # Print the cross-validation scores\n",
    "    # print(\"Cross-validation scores:\", cv_scores)\n",
    "    # print(\"Mean accuracy:\", cv_scores.mean())\n",
    "    # with open(output_file_name, \"a\") as f: print('mean accuracy', cv_scores.mean() , file = f)\n",
    "\n",
    "    ada_tr_time_taken= time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "\n",
    "    # Assuming 'model' is your trained model\n",
    "    # joblib.dump(ada, 'ada_base_model.joblib')\n",
    "\n",
    "\n",
    "if use_model_rf == 1 and load_model_rf == 0:\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training RF')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('Training RF', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #RF\n",
    "    start = time.time()\n",
    "    model_rf = rf.fit(X_train,y_train)\n",
    "    end = time.time()\n",
    "\n",
    "    # # Create the StratifiedKFold object\n",
    "    # stratified_kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "    # # Perform cross-validation\n",
    "    # cv_scores = cross_val_score(model_rf, X_train, y_train, cv=stratified_kfold, scoring='accuracy')\n",
    "    # # Print the cross-validation scores\n",
    "    # print(\"Cross-validation scores:\", cv_scores)\n",
    "    # print(\"Mean accuracy:\", cv_scores.mean())\n",
    "    # with open(output_file_name, \"a\") as f: print('mean accuracy', cv_scores.mean() , file = f)\n",
    "\n",
    "\n",
    "    rf_tr_time_taken = time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    # joblib.dump(model_rf, 'rf_base_model.joblib')\n",
    "\n",
    "if use_model_svm == 1 and load_model_svm == 0:\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training SVM')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training SVM', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #SVM\n",
    "\n",
    "    start = time.time()\n",
    "    clf.fit(X_train, y_train)\n",
    "    end = time.time()\n",
    "    # clf.score(X_train, y_train)\n",
    "    svm_tr_time_taken= time_taken = end - start\n",
    "\n",
    "    # # Create the StratifiedKFold object\n",
    "    # stratified_kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "    # # Perform cross-validation\n",
    "    # cv_scores = cross_val_score(clf, X_train, y_train, cv=stratified_kfold, scoring='accuracy')\n",
    "    # # Print the cross-validation scores\n",
    "    # print(\"Cross-validation scores:\", cv_scores)\n",
    "    # print(\"Mean accuracy:\", cv_scores.mean())\n",
    "    # with open(output_file_name, \"a\") as f: print('mean accuracy', cv_scores.mean() , file = f)\n",
    "\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    # joblib.dump(clf, 'svm_base_model.joblib')\n",
    "\n",
    "\n",
    "if use_model_knn == 1 and load_model_knn == 0:\n",
    "\n",
    "    #KNN\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training KNN')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training KNN', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    start = time.time()\n",
    "    knn_clf.fit(X_train,y_train)\n",
    "    end = time.time()\n",
    "\n",
    "\n",
    "    # # Create the StratifiedKFold object\n",
    "    # stratified_kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "    # # Perform cross-validation\n",
    "    # cv_scores = cross_val_score(knn_clf, X_train, y_train, cv=stratified_kfold, scoring='accuracy')\n",
    "    # # Print the cross-validation scores\n",
    "    # print(\"Cross-validation scores:\", cv_scores)\n",
    "    # print(\"Mean accuracy:\", cv_scores.mean())\n",
    "    # with open(output_file_name, \"a\") as f: print('mean accuracy', cv_scores.mean() , file = f)\n",
    "\n",
    "\n",
    "    knn_tr_time_taken = time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    # joblib.dump(knn_clf, 'knn_base_model.joblib')\n",
    "\n",
    "\n",
    "if use_model_lgbm == 1 and load_model_lgbm == 0:\n",
    "\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training LGBM')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training LGBM', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    start = time.time()\n",
    "    lgbm.fit(X_train, y_train)\n",
    "    end = time.time()\n",
    "\n",
    "    # # Create the StratifiedKFold object\n",
    "    # stratified_kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "    # # Perform cross-validation\n",
    "    # cv_scores = cross_val_score(lgbm, X_train, y_train, cv=stratified_kfold, scoring='accuracy')\n",
    "    # # Print the cross-validation scores\n",
    "    # print(\"Cross-validation scores:\", cv_scores)\n",
    "    # print(\"Mean accuracy:\", cv_scores.mean())\n",
    "    # with open(output_file_name, \"a\") as f: print('mean accuracy', cv_scores.mean() , file = f)\n",
    "\n",
    "    lgbm_tr_time_taken = time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    # joblib.dump(lgbm, 'lgbm_base_model.joblib')\n",
    "\n",
    "if use_model_mlp == 1 and load_model_mlp == 0:\n",
    "\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training MLP')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training MLP', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "    start = time.time()\n",
    "    MLP = mlp.fit(X_train, y_train)\n",
    "    end = time.time()\n",
    "\n",
    "    # # Create the StratifiedKFold object\n",
    "    # stratified_kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "    # # Perform cross-validation\n",
    "    # cv_scores = cross_val_score(MLP, X_train, y_train, cv=stratified_kfold, scoring='accuracy')\n",
    "    # # Print the cross-validation scores\n",
    "    # print(\"Cross-validation scores:\", cv_scores)\n",
    "    # print(\"Mean accuracy:\", cv_scores.mean())\n",
    "    # with open(output_file_name, \"a\") as f: print('mean accuracy', cv_scores.mean() , file = f)\n",
    "\n",
    "    mlp_tr_time_taken= time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    # joblib.dump(MLP, 'mlp_base_model.joblib')\n",
    "\n",
    "\n",
    "if use_model_dnn == 1 and load_model_dnn == 0:\n",
    "    from keras.callbacks import EarlyStopping\n",
    "\n",
    "    # Define EarlyStopping callback\n",
    "    early_stopping = EarlyStopping(monitor='val_accuracy', patience=10, restore_best_weights=True)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training DNN')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training DNN', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    # Convert Y_test back to its original format\n",
    "    # y_test = np.argmax(Y_test, axis=1)\n",
    "\n",
    "    # Start the timer\n",
    "    start = time.time()\n",
    "    # dnn.fit(X_train, y_train, epochs=epochs, batch_size=batch_size)\n",
    "    dnn.fit(X_train, y_train, epochs=epochs, batch_size=batch_size,validation_split=0.2, callbacks=[early_stopping])\n",
    "\n",
    "    # End the timer\n",
    "    end = time.time()\n",
    "\n",
    "    # # Create the StratifiedKFold object\n",
    "    # stratified_kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "    # # Perform cross-validation\n",
    "    # cv_scores = cross_val_score(dnn, X_train, y_train, cv=stratified_kfold, scoring='accuracy')\n",
    "    # # Print the cross-validation scores\n",
    "    # print(\"Cross-validation scores:\", cv_scores)\n",
    "    # print(\"Mean accuracy:\", cv_scores.mean())\n",
    "    # with open(output_file_name, \"a\") as f: print('mean accuracy', cv_scores.mean() , file = f)\n",
    "\n",
    "\n",
    "    dnn_tr_time_taken= time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    # dnn.save(\"DNN_base_model.h5\")\n",
    "\n",
    "    # Calculate the time taken and print it out\n",
    "    # print(f'Time taken for training: {time_taken} seconds')\n",
    "\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from keras.models import Sequential\n",
    "# from keras.layers import Dense\n",
    "# from keras.wrappers.scikit_learn import KerasClassifier\n",
    "# from sklearn.model_selection import GridSearchCV\n",
    "# from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "# # Define your Keras model as a function\n",
    "# def create_model(optimizer='adam', hidden_layer_size=16):\n",
    "#     # model = Sequential()\n",
    "#     # model.add(Dense(hidden_layer_size, input_dim=input_size, activation='relu'))\n",
    "#     # model.add(Dense(1, activation='sigmoid'))\n",
    "#     # model.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=['accuracy'])\n",
    "\n",
    "        \n",
    "#     dnn = tf.keras.Sequential()\n",
    "\n",
    "#     # Input layer\n",
    "#     dnn.add(tf.keras.Input(shape=(num_columns,)))\n",
    "\n",
    "#     # Dense layers with dropout\n",
    "#     dnn.add(tf.keras.layers.Dense(nodes))\n",
    "#     dnn.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "#     dnn.add(tf.keras.layers.Dense(nodes))\n",
    "#     dnn.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "#     dnn.add(tf.keras.layers.Dense(nodes))\n",
    "#     dnn.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "#     dnn.add(tf.keras.layers.Dense(nodes))\n",
    "#     dnn.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "#     dnn.add(tf.keras.layers.Dense(nodes))\n",
    "#     dnn.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "\n",
    "#     # Output layer\n",
    "#     dnn.add(tf.keras.layers.Dense(out_layer))\n",
    "\n",
    "\n",
    "\n",
    "#     dnn.compile(optimizer=optimizer, loss=loss)\n",
    "\n",
    "#     dnn.summary()\n",
    "#     return dnn\n",
    "\n",
    "# # Create a KerasClassifier\n",
    "# dnn = KerasClassifier(build_fn=create_model, epochs=10, batch_size=32, verbose=0)\n",
    "\n",
    "# # Define the parameter grid for GridSearchCV\n",
    "# param_grid = {\n",
    "#     'optimizer': ['adam', 'sgd'],\n",
    "#     'hidden_layer_size': [8, 16, 32]\n",
    "# }\n",
    "\n",
    "# # Create the StratifiedKFold\n",
    "# cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n",
    "\n",
    "# # Create GridSearchCV\n",
    "# grid = GridSearchCV(estimator=dnn, param_grid=param_grid, cv=cv, scoring='accuracy')\n",
    "# grid_result = grid.fit(X_train, y_train)\n",
    "\n",
    "# # Print the best parameters and best accuracy\n",
    "# print(\"Best Parameters: \", grid_result.best_params_)\n",
    "# print(\"Best Accuracy: \", grid_result.best_score_)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "# stratified_kfold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Loading Models\n",
    "from tensorflow.keras.models import load_model\n",
    "\n",
    "if load_model_ada == 1:\n",
    "    ada = joblib.load('ada_base_model.joblib')\n",
    "\n",
    "if load_model_svm == 1:\n",
    "    clf =  joblib.load('svm_base_model.joblib')\n",
    "\n",
    "if load_model_dnn == 1:\n",
    "    dnn = load_model(\"DNN_base_model.h5\")\n",
    "\n",
    "if load_model_knn == 1:\n",
    "    knn_clf = joblib.load('knn_base_model.joblib')\n",
    "\n",
    "if load_model_mlp == 1:\n",
    "    MLP = joblib.load('mlp_base_model.joblib')\n",
    "\n",
    "if load_model_rf == 1:\n",
    "    rf = joblib.load('rf_base_model.joblib')\n",
    "\n",
    "if load_model_lgbm == 1:\n",
    "    lgbm = joblib.load('lgbm_base_model.joblib')\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make predictions on the test data\n",
    "# preds_svm = clf.predict(X_test)\n",
    "\n",
    "\n",
    "\n",
    "# y_scores = y_pred\n",
    "# y_true = y_test\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Base leaners predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Prediction RF\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Prediction SVM\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Prediction LGBM\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Prediction DNN\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Prediction ADA\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Prediction MLP\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Prediction KNN\n",
      "---------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "from sklearn.calibration import CalibratedClassifierCV\n",
    "with open(output_file_name, \"a\") as f: print('Generating Predictions', file = f)\n",
    "\n",
    "if use_model_rf == 1:\n",
    "\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Prediction RF')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Prediction RF', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #RF\n",
    "    start = time.time()\n",
    "    preds_rf = rf.predict(X_test)\n",
    "    preds_rf_prob = rf.predict_proba(X_test)\n",
    "    end = time.time()\n",
    "    rf_pr_time_taken=  time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "if use_model_svm == 1:\n",
    "\n",
    "    print('Prediction SVM')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Prediction SVM', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #SVM\n",
    "    start = time.time()\n",
    "    preds_svm = clf.predict(X_test)\n",
    "    # preds_svm_prob = clf.predict_proba(X_test)\n",
    "\n",
    "    #Since SVM does not deal with prob by nature we use a meta learner\n",
    "    # https://stackoverflow.com/questions/55250963/how-to-get-probabilities-for-sgdclassifier-linearsvm\n",
    "\n",
    "    model = CalibratedClassifierCV(clf)\n",
    "\n",
    "    model.fit(X, y)\n",
    "    preds_svm_prob = model.predict_proba(X)\n",
    "\n",
    "    end = time.time()\n",
    "    svm_pr_time_taken = time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "if use_model_lgbm == 1:\n",
    "\n",
    "    print('Prediction LGBM')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Prediction LGBM', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #LGBM\n",
    "    start = time.time()\n",
    "    preds_lgbm = lgbm.predict(X_test)\n",
    "    preds_lgbm_prob = lgbm.predict_proba(X_test)\n",
    "\n",
    "    end = time.time()\n",
    "    lgbm_pr_time_taken=time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "if use_model_dnn == 1:\n",
    "\n",
    "    print('Prediction DNN')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Prediction DNN', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #DNN\n",
    "    start = time.time()\n",
    "    pred_dnn = dnn.predict(X_test)\n",
    "    preds_dnn_prob = pred_dnn\n",
    "    preds_dnn = np.argmax(pred_dnn,axis = 1)\n",
    "    end = time.time()\n",
    "    dnn_pr_time_taken=time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "if use_model_ada == 1:\n",
    "\n",
    "    print('Prediction ADA')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Prediction ADA', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    #ADA\n",
    "    start = time.time()\n",
    "    preds_ada = ada.predict(X_test)\n",
    "    preds_ada_prob = ada.predict_proba(X_test)\n",
    "\n",
    "    end = time.time()\n",
    "    ada_pr_time_taken=time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Prediction MLP')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Prediction MLP', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "if use_model_mlp == 1:\n",
    "\n",
    "    #MLP\n",
    "    start = time.time()\n",
    "    y_pred = MLP.predict_proba(X_test)\n",
    "    preds_mlp_prob = y_pred\n",
    "    preds_mlp = np.argmax(y_pred,axis = 1)\n",
    "    end = time.time()\n",
    "    mlp_pr_time_taken=time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Prediction KNN')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Prediction KNN', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "\n",
    "if use_model_knn == 1:\n",
    "\n",
    "    #KNN\n",
    "    start = time.time()\n",
    "    preds_knn =knn_clf.predict(X_test)\n",
    "    preds_knn_prob =knn_clf.predict_proba(X_test)\n",
    "\n",
    "    preds_knn\n",
    "    end = time.time()\n",
    "    knn_pr_time_taken=time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from sklearn.calibration import CalibratedClassifierCV\n",
    "# model = CalibratedClassifierCV(clf)\n",
    "\n",
    "# model.fit(X, y)\n",
    "# preds_svm_prob = model.predict_proba(X)\n",
    "\n",
    "# print(preds_ada_prob)\n",
    "# print(preds_knn_prob)\n",
    "# print(preds_dnn_prob)\n",
    "# print(preds_mlp_prob)\n",
    "# print(preds_rf_prob)\n",
    "# print(preds_svm_prob)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.83138421 0.01832765 0.00154958 ... 0.00154843 0.00766981 0.13185052]\n",
      " [0.79430046 0.01832765 0.00154927 ... 0.00154957 0.00747006 0.16933294]\n",
      " [0.83114608 0.01832819 0.00154413 ... 0.00154771 0.00766768 0.13209853]\n",
      " ...\n",
      " [0.04813801 0.01922512 0.00290377 ... 0.2968983  0.00654246 0.61974989]\n",
      " [0.04815442 0.0192126  0.00289403 ... 0.29788007 0.006535   0.61878889]\n",
      " [0.04799246 0.01934697 0.00299559 ... 0.28788954 0.00661083 0.62855378]]\n",
      "[0 0 0 ... 6 6 6]\n",
      "[6 6 6 ... 6 6 6]\n"
     ]
    }
   ],
   "source": [
    "print(preds_svm_prob)\n",
    "preds_3 = np.argmax(preds_svm_prob,axis = 1)\n",
    "print(preds_3)\n",
    "\n",
    "print(preds_svm)\n",
    "# print(y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### METRICS - Base Learners"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.metrics import balanced_accuracy_score\n",
    "from sklearn.metrics import matthews_corrcoef\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "\n",
    "\n",
    "# >>> \n",
    "# >>> roc_auc_score(y, clf.predict_proba(X)[:, 1])\n",
    "# 0.99...\n",
    "# >>> roc_auc_score(y, clf.decision_function(X))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### RF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "# y_test\n",
    "# pred_label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0       1    2      3        4    5    6\n",
      "0  12028.0     0.0  0.0    0.0    656.0  0.0  0.0\n",
      "1      4.0  5223.0  0.0    0.0     47.0  0.0  0.0\n",
      "2      1.0     0.0  0.0    0.0    418.0  0.0  0.0\n",
      "3      0.0     0.0  8.0    0.0    466.0  0.0  0.0\n",
      "4      0.0     0.0  0.0    0.0    449.0  0.0  0.0\n",
      "5      0.0     0.0  0.0  190.0    281.0  0.0  0.0\n",
      "6    180.0    20.0  0.0    0.0  37573.0  0.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9561726678715418\n",
      "Precision total:  0.7032789805275862\n",
      "Recall total:  0.4790844424412081\n",
      "F1 total:  0.505020308018976\n",
      "BACC total:  0.4790844424412081\n",
      "MCC total:  0.9130332538595067\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "#RF\n",
    "if use_model_rf == 1:\n",
    "    # start = time.time()\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('RF base model', file = f)\n",
    "\n",
    "    pred_label = preds_rf\n",
    "    name = 'rf'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "    globals()[f\"{name}_acc_00\"] = Acc\n",
    "    globals()[f\"{name}_pre_00\"] = Precision\n",
    "    globals()[f\"{name}_rec_00\"] = Recall\n",
    "    globals()[f\"{name}_f1_00\"] = F1\n",
    "    globals()[f\"{name}_bacc_00\"] = BACC\n",
    "    globals()[f\"{name}_mcc_00\"] = MCC\n",
    "    globals()[f\"{name}_time_00\"] = rf_pr_time_taken + rf_tr_time_taken\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "         0        1    2    3    4    5    6\n",
      "0  12027.0    657.0  0.0  0.0  0.0  0.0  0.0\n",
      "1     10.0   5264.0  0.0  0.0  0.0  0.0  0.0\n",
      "2      1.0    418.0  0.0  0.0  0.0  0.0  0.0\n",
      "3      0.0    474.0  0.0  0.0  0.0  0.0  0.0\n",
      "4      0.0    449.0  0.0  0.0  0.0  0.0  0.0\n",
      "5     33.0    438.0  0.0  0.0  0.0  0.0  0.0\n",
      "6    673.0  37100.0  0.0  0.0  0.0  0.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.8537293201723898\n",
      "Precision total:  0.25312331853645414\n",
      "Recall total:  0.27576935722688567\n",
      "F1 total:  0.263509127637199\n",
      "BACC total:  0.27576935722688567\n",
      "MCC total:  0.6994586416382897\n"
     ]
    }
   ],
   "source": [
    "#DNN\n",
    "if use_model_dnn == 1:\n",
    "    start = time.time()\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('DNN base model', file = f)\n",
    "\n",
    "\n",
    "    pred_label = preds_dnn\n",
    "    name = 'dnn'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "    globals()[f\"{name}_acc_00\"] = Acc\n",
    "    globals()[f\"{name}_pre_00\"] = Precision\n",
    "    globals()[f\"{name}_rec_00\"] = Recall\n",
    "    globals()[f\"{name}_f1_00\"] = F1\n",
    "    globals()[f\"{name}_bacc_00\"] = BACC\n",
    "    globals()[f\"{name}_mcc_00\"] = MCC\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    globals()[f\"{name}_time_00\"] = dnn_pr_time_taken + dnn_tr_time_taken"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0      1     2        3    4    5    6\n",
      "0  7974.0    0.0   0.0   4710.0  0.0  0.0  0.0\n",
      "1     4.0   80.0   0.0   5190.0  0.0  0.0  0.0\n",
      "2     0.0  419.0   0.0      0.0  0.0  0.0  0.0\n",
      "3     0.0    0.0   0.0    474.0  0.0  0.0  0.0\n",
      "4     0.0    0.0   0.0    449.0  0.0  0.0  0.0\n",
      "5     0.0    0.0  49.0    422.0  0.0  0.0  0.0\n",
      "6    49.0  268.0   1.0  37455.0  0.0  0.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy total:  0.7975983595161963\n",
      "Precision total:  0.4698254310937875\n",
      "Recall total:  0.3891830420640798\n",
      "F1 total:  0.3615691553814126\n",
      "BACC total:  0.3891830420640798\n",
      "MCC total:  0.5744246506252494\n"
     ]
    }
   ],
   "source": [
    "#ADA\n",
    "if use_model_ada == 1:\n",
    "    start = time.time()\n",
    "    \n",
    "    pred_label = preds_ada\n",
    "    name = 'ada'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "    globals()[f\"{name}_acc_00\"] = Acc\n",
    "    globals()[f\"{name}_pre_00\"] = Precision\n",
    "    globals()[f\"{name}_rec_00\"] = Recall\n",
    "    globals()[f\"{name}_f1_00\"] = F1\n",
    "    globals()[f\"{name}_bacc_00\"] = BACC\n",
    "    globals()[f\"{name}_mcc_00\"] = MCC\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    globals()[f\"{name}_time_00\"] = ada_pr_time_taken + ada_tr_time_taken"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0       1     2    3        4    5    6\n",
      "0  11368.0    61.0   0.0  0.0   1255.0  0.0  0.0\n",
      "1      6.0  5204.0   0.0  0.0     64.0  0.0  0.0\n",
      "2      1.0     0.0   0.0  0.0    418.0  0.0  0.0\n",
      "3      0.0   114.0   0.0  0.0    360.0  0.0  0.0\n",
      "4      0.0     0.0   0.0  0.0    449.0  0.0  0.0\n",
      "5     71.0     0.0  56.0  1.0    343.0  0.0  0.0\n",
      "6    617.0  1029.0  14.0  2.0  36111.0  0.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.915525510913388\n",
      "Precision total:  0.38291696391955704\n",
      "Recall total:  0.40556784285449793\n",
      "F1 total:  0.39291461481893897\n",
      "BACC total:  0.40556784285449793\n",
      "MCC total:  0.8319201808015049\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "#SVM\n",
    "if use_model_svm == 1:\n",
    "    start = time.time()\n",
    "\n",
    "    pred_label = preds_svm\n",
    "    name = 'svm'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "    globals()[f\"{name}_acc_00\"] = Acc\n",
    "    globals()[f\"{name}_pre_00\"] = Precision\n",
    "    globals()[f\"{name}_rec_00\"] = Recall\n",
    "    globals()[f\"{name}_f1_00\"] = F1\n",
    "    globals()[f\"{name}_bacc_00\"] = BACC\n",
    "    globals()[f\"{name}_mcc_00\"] = MCC\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    globals()[f\"{name}_time_00\"] = svm_pr_time_taken + svm_tr_time_taken"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0       1      2      3      4      5        6\n",
      "0  12656.0     0.0    0.0    0.0    0.0    0.0     28.0\n",
      "1      5.0  5109.0    0.0    0.0    0.0    0.0    160.0\n",
      "2      0.0     0.0  417.0    0.0    0.0    0.0      2.0\n",
      "3      0.0     0.0    0.0  449.0    0.0    0.0     25.0\n",
      "4      0.0     0.0    1.0    0.0  447.0    0.0      1.0\n",
      "5      0.0     0.0    0.0    0.0    0.0  471.0      0.0\n",
      "6     38.0   350.0    3.0   57.0   10.0   11.0  37304.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9879917975809815\n",
      "Precision total:  0.9656985494985014\n",
      "Recall total:  0.984588633976266\n",
      "F1 total:  0.9749311924824325\n",
      "BACC total:  0.984588633976266\n",
      "MCC total:  0.9766963510272219\n"
     ]
    }
   ],
   "source": [
    "#KNN\n",
    "if use_model_knn == 1:\n",
    "    start = time.time()\n",
    "    pred_label = preds_knn\n",
    "    name = 'knn'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "    globals()[f\"{name}_acc_00\"] = Acc\n",
    "    globals()[f\"{name}_pre_00\"] = Precision\n",
    "    globals()[f\"{name}_rec_00\"] = Recall\n",
    "    globals()[f\"{name}_f1_00\"] = F1\n",
    "    globals()[f\"{name}_bacc_00\"] = BACC\n",
    "    globals()[f\"{name}_mcc_00\"] = MCC\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    globals()[f\"{name}_time_00\"] = knn_pr_time_taken + knn_tr_time_taken"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0       1      2      3      4      5        6\n",
      "0  12578.0     0.0    0.0    0.0    0.0    0.0    106.0\n",
      "1      3.0  5267.0    0.0    0.0    0.0    0.0      4.0\n",
      "2      1.0     0.0  415.0    0.0    0.0    0.0      3.0\n",
      "3      0.0     0.0    0.0  311.0    0.0    0.0    163.0\n",
      "4      0.0     1.0   27.0    0.0  420.0    0.0      1.0\n",
      "5      0.0     0.0    0.0    0.0    0.0  471.0      0.0\n",
      "6    162.0   529.0    1.0   23.0   95.0    4.0  36959.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9804844988182956\n",
      "Precision total:  0.9375934085480729\n",
      "Recall total:  0.935821370794074\n",
      "F1 total:  0.9322961544092099\n",
      "BACC total:  0.935821370794074\n",
      "MCC total:  0.9624592600740894\n"
     ]
    }
   ],
   "source": [
    "#MLP\n",
    "if use_model_mlp == 1:\n",
    "    start = time.time()\n",
    "    pred_label = preds_mlp\n",
    "    name = 'mlp'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "    globals()[f\"{name}_acc_00\"] = Acc\n",
    "    globals()[f\"{name}_pre_00\"] = Precision\n",
    "    globals()[f\"{name}_rec_00\"] = Recall\n",
    "    globals()[f\"{name}_f1_00\"] = F1\n",
    "    globals()[f\"{name}_bacc_00\"] = BACC\n",
    "    globals()[f\"{name}_mcc_00\"] = MCC\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    globals()[f\"{name}_time_00\"] = mlp_pr_time_taken + mlp_tr_time_taken"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0       1      2      3      4      5        6\n",
      "0  12682.0     0.0    0.0    0.0    0.0    0.0      2.0\n",
      "1      0.0  5269.0    0.0    0.0    0.0    0.0      5.0\n",
      "2      0.0     0.0  418.0    0.0    0.0    0.0      1.0\n",
      "3      0.0     0.0    0.0  467.0    0.0    0.0      7.0\n",
      "4      0.0     0.0    0.0    0.0  449.0    0.0      0.0\n",
      "5      0.0     0.0    0.0    0.0    0.0  470.0      1.0\n",
      "6     10.0     9.0    1.0    9.0    0.0    0.0  37744.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9992179897122202\n",
      "Precision total:  0.9965412834382235\n",
      "Recall total:  0.9969784028957874\n",
      "F1 total:  0.9967590205828405\n",
      "BACC total:  0.9969784028957874\n",
      "MCC total:  0.9984727039083049\n"
     ]
    }
   ],
   "source": [
    "#lgbm\n",
    "start_lgbm = time.time()\n",
    "if use_model_lgbm == 1:\n",
    "\n",
    "    pred_label = preds_lgbm\n",
    "    name = 'lgbm'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "    globals()[f\"{name}_acc_00\"] = Acc\n",
    "    globals()[f\"{name}_pre_00\"] = Precision\n",
    "    globals()[f\"{name}_rec_00\"] = Recall\n",
    "    globals()[f\"{name}_f1_00\"] = F1\n",
    "    globals()[f\"{name}_bacc_00\"] = BACC\n",
    "    globals()[f\"{name}_mcc_00\"] = MCC\n",
    "    end = time.time()\n",
    "    time_taken = end - start_lgbm\n",
    "    globals()[f\"{name}_time_00\"] = lgbm_pr_time_taken + lgbm_tr_time_taken"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Decision Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0       1      2      3      4      5        6\n",
      "0  12674.0     0.0    0.0    0.0    0.0    0.0     10.0\n",
      "1      0.0  5257.0    0.0    0.0    0.0    0.0     17.0\n",
      "2      0.0     0.0  417.0    0.0    0.0    0.0      2.0\n",
      "3      0.0     0.0    0.0  459.0    0.0    0.0     15.0\n",
      "4      0.0     0.0    0.0    0.0  448.0    0.0      1.0\n",
      "5      0.0     0.0    0.0    0.0    0.0  470.0      1.0\n",
      "6     15.0    11.0    4.0   19.0    4.0    5.0  37715.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.998192687334909\n",
      "Precision total:  0.9895550845384244\n",
      "Recall total:  0.9933833719898196\n",
      "F1 total:  0.9914618239473202\n",
      "BACC total:  0.9933833719898196\n",
      "MCC total:  0.9964702241880841\n"
     ]
    }
   ],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "start = time.time()\n",
    "\n",
    "# Create a Decision Tree Classifier\n",
    "dt_classifier = DecisionTreeClassifier(random_state=42)\n",
    "# Train the classifier on the training data\n",
    "dt_classifier.fit(X_train, y_train)\n",
    "# Make predictions on the test data\n",
    "preds_dt = dt_classifier.predict(X_test)\n",
    "# Evaluate the accuracy of the model\n",
    "preds_dt_prob = dt_classifier.predict_proba(X_test)\n",
    "\n",
    "\n",
    "pred_label = preds_dt\n",
    "name = 'dt'\n",
    "metrics = confusion_metrics(name, pred_label, y_test)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "globals()[f\"{name}_acc_00\"] = Acc\n",
    "globals()[f\"{name}_pre_00\"] = Precision\n",
    "globals()[f\"{name}_rec_00\"] = Recall\n",
    "globals()[f\"{name}_f1_00\"] = F1\n",
    "globals()[f\"{name}_bacc_00\"] = BACC\n",
    "globals()[f\"{name}_mcc_00\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_00\"] = time_taken"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CATBOOST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:\tlearn: 1.4334148\ttest: 1.4344829\tbest: 1.4344829 (0)\ttotal: 118ms\tremaining: 11.7s\n",
      "10:\tlearn: 0.3940317\ttest: 0.3958580\tbest: 0.3958580 (10)\ttotal: 595ms\tremaining: 4.81s\n",
      "20:\tlearn: 0.1761320\ttest: 0.1776643\tbest: 0.1776643 (20)\ttotal: 1.06s\tremaining: 3.97s\n",
      "30:\tlearn: 0.0964146\ttest: 0.0977583\tbest: 0.0977583 (30)\ttotal: 1.5s\tremaining: 3.34s\n",
      "40:\tlearn: 0.0600479\ttest: 0.0607680\tbest: 0.0607680 (40)\ttotal: 1.95s\tremaining: 2.81s\n",
      "50:\tlearn: 0.0428544\ttest: 0.0432272\tbest: 0.0432272 (50)\ttotal: 2.38s\tremaining: 2.29s\n",
      "60:\tlearn: 0.0335981\ttest: 0.0337964\tbest: 0.0337964 (60)\ttotal: 2.82s\tremaining: 1.8s\n",
      "70:\tlearn: 0.0269313\ttest: 0.0269081\tbest: 0.0269081 (70)\ttotal: 3.27s\tremaining: 1.33s\n",
      "80:\tlearn: 0.0225417\ttest: 0.0223954\tbest: 0.0223954 (80)\ttotal: 3.7s\tremaining: 869ms\n",
      "90:\tlearn: 0.0200408\ttest: 0.0197904\tbest: 0.0197904 (90)\ttotal: 4.13s\tremaining: 409ms\n",
      "99:\tlearn: 0.0177681\ttest: 0.0174797\tbest: 0.0174797 (99)\ttotal: 4.53s\tremaining: 0us\n",
      "\n",
      "bestTest = 0.01747968851\n",
      "bestIteration = 99\n",
      "\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0       1      2      3      4      5        6\n",
      "0  12648.0     0.0    0.0    0.0    1.0    0.0     35.0\n",
      "1      4.0  5261.0    0.0    0.0    2.0    0.0      7.0\n",
      "2      0.0     0.0  417.0    0.0    0.0    0.0      2.0\n",
      "3      0.0     0.0    0.0  451.0    0.0    0.0     23.0\n",
      "4      0.0     0.0    0.0    0.0  442.0    0.0      7.0\n",
      "5      0.0     0.0    0.0    0.0    0.0  468.0      3.0\n",
      "6     38.0    12.0    0.0   15.0    0.0    0.0  37708.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9974106770471292\n",
      "Precision total:  0.9933494718613088\n",
      "Recall total:  0.9882457066934494\n",
      "F1 total:  0.9907825495819125\n",
      "BACC total:  0.9882457066934494\n",
      "MCC total:  0.9949401148760677\n"
     ]
    }
   ],
   "source": [
    "import catboost\n",
    "start = time.time()\n",
    "cat_00 = catboost.CatBoostClassifier(iterations=100, depth=6, learning_rate=0.1, loss_function='MultiClass', custom_metric='Accuracy')\n",
    "\n",
    "# Fit the model\n",
    "cat_00.fit(X_train, y_train, eval_set=(X_test, y_test), verbose=10)\n",
    "\n",
    "# Make predictions on the test set\n",
    "preds_cat = cat_00.predict(X_test)\n",
    "preds_cat_prob = cat_00.predict_proba(X_test)\n",
    "preds_cat = np.squeeze(preds_cat)\n",
    "\n",
    "\n",
    "if 1 == 1:\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Catboost base model', file = f)\n",
    "\n",
    "\n",
    "    \n",
    "\n",
    "    pred_label = preds_cat\n",
    "    \n",
    "    \n",
    "\n",
    "    # pred_label = y_pred\n",
    "\n",
    "    name = 'cat'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "    globals()[f\"{name}_acc_00\"] = Acc\n",
    "    globals()[f\"{name}_pre_00\"] = Precision\n",
    "    globals()[f\"{name}_rec_00\"] = Recall\n",
    "    globals()[f\"{name}_f1_00\"] = F1\n",
    "    globals()[f\"{name}_bacc_00\"] = BACC\n",
    "    globals()[f\"{name}_mcc_00\"] = MCC\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    globals()[f\"{name}_time_00\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [],
   "source": [
    "import xgboost as xgb\n",
    "start = time.time()\n",
    "# from sklearn.model_selection import train_test_split\n",
    "# from sklearn.metrics import accuracy_score\n",
    "\n",
    "# Assuming you have your features and labels as X and y\n",
    "# X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n",
    "\n",
    "# Create a DMatrix for XGBoost\n",
    "dtrain = xgb.DMatrix(X_train, label=y_train)\n",
    "dtest = xgb.DMatrix(X_test, label=y_test)\n",
    "\n",
    "# Set XGBoost parameters\n",
    "params = {\n",
    "    'objective': 'multi:softmax',  # for multi-class classification\n",
    "    'num_class': 7,  # specify the number of classes\n",
    "    'max_depth': 3,\n",
    "    'learning_rate': 0.1,\n",
    "    'eval_metric': 'mlogloss'  # metric for multi-class classification\n",
    "}\n",
    "\n",
    "# Train the XGBoost model\n",
    "num_round = 100\n",
    "xgb_00 = xgb.train(params, dtrain, num_round)\n",
    "\n",
    "# Make predictions on the test set\n",
    "preds_xgb = xgb_00.predict(dtest)\n",
    "# preds_xgb_prob = xgb_00.predict_proba(dtest)\n",
    "\n",
    "\n",
    "# Get class probabilities\n",
    "# Assuming binary classification, get the probability for the positive class (class 1)\n",
    "preds_xgb_margin = xgb_00.predict(dtest, output_margin=True)\n",
    "preds_xgb_prob = 1 / (1 + np.exp(-preds_xgb_margin))\n",
    "\n",
    "# Print or use positive_class_probabilities as needed\n",
    "# print(positive_class_probabilities)\n",
    "\n",
    "\n",
    "# Convert predicted probabilities to class labels (if necessary)\n",
    "# y_pred_labels = [round(value) for value in y_pred]\n",
    "\n",
    "# Evaluate the accuracy\n",
    "# accuracy = accuracy_score(y_test, y_pred)\n",
    "# print(\"Accuracy: %.2f%%\" % (accuracy * 100.0))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0.0     1.0    2.0    3.0    4.0    5.0      6.0\n",
      "0.0  12651.0     0.0    0.0    0.0    2.0    0.0     31.0\n",
      "1.0      4.0  5268.0    0.0    0.0    0.0    0.0      2.0\n",
      "2.0      0.0     0.0  418.0    0.0    0.0    0.0      1.0\n",
      "3.0      0.0     0.0    0.0  452.0    0.0    0.0     22.0\n",
      "4.0      5.0     0.0    0.0    0.0  435.0    0.0      9.0\n",
      "5.0      0.0     0.0    0.0    0.0    0.0  466.0      5.0\n",
      "6.0     58.0    10.0    4.0   16.0    0.0    0.0  37685.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.997063116919227\n",
      "Precision total:  0.9918199705337617\n",
      "Recall total:  0.9861906693053245\n",
      "F1 total:  0.9889678837786372\n",
      "BACC total:  0.9861906693053245\n",
      "MCC total:  0.994263852949557\n"
     ]
    }
   ],
   "source": [
    "\n",
    "if 1 == 1:\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('xgboost base model', file = f)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    pred_label = preds_xgb\n",
    "    # pred_label = label[ypred]\n",
    "    name = 'xgb'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "    globals()[f\"{name}_acc_00\"] = Acc\n",
    "    globals()[f\"{name}_pre_00\"] = Precision\n",
    "    globals()[f\"{name}_rec_00\"] = Recall\n",
    "    globals()[f\"{name}_f1_00\"] = F1\n",
    "    globals()[f\"{name}_bacc_00\"] = BACC\n",
    "    globals()[f\"{name}_mcc_00\"] = MCC\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    globals()[f\"{name}_time_00\"] = time_taken"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining Logistic Regression Model\n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "Training LR \n",
      "---------------------------------------------------------------------------------\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0       1     2    3     4      5        6\n",
      "0  11710.0    65.0   2.0  0.0   0.0    3.0    904.0\n",
      "1      4.0  5226.0   0.0  0.0   0.0    0.0     44.0\n",
      "2      1.0     0.0   0.0  0.0   0.0    0.0    418.0\n",
      "3      0.0   114.0   0.0  4.0   0.0    0.0    356.0\n",
      "4      0.0     0.0   0.0  0.0   0.0    0.0    449.0\n",
      "5     32.0     0.0   0.0  0.0   0.0  393.0     46.0\n",
      "6    631.0   924.0  13.0  8.0  63.0   28.0  36106.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9286632837480884\n",
      "Precision total:  0.5677322709428243\n",
      "Recall total:  0.5304015376210376\n",
      "F1 total:  0.5255594865089178\n",
      "BACC total:  0.5304015376210376\n",
      "MCC total:  0.8594190961299031\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "lbfgs failed to converge (status=2):\n",
      "ABNORMAL_TERMINATION_IN_LNSRCH.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "#Logistic Regression\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining Logistic Regression Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "logreg_00 = LogisticRegression()\n",
    "\n",
    "if 1 == 1 and 0 == 0:\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    print('Training LR ')\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "    with open(output_file_name, \"a\") as f: print('Training LR', file = f)\n",
    "    print('---------------------------------------------------------------------------------')\n",
    "    start_lr = start = time.time()\n",
    "    logreg_00.fit(X_train,y_train)\n",
    "    end = time.time()\n",
    "\n",
    "\n",
    "    # # Create the StratifiedKFold object\n",
    "    # stratified_kfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
    "    # # Perform cross-validation\n",
    "    # cv_scores = cross_val_score(knn_clf, X_train, y_train, cv=stratified_kfold, scoring='accuracy')\n",
    "    # # Print the cross-validation scores\n",
    "    # print(\"Cross-validation scores:\", cv_scores)\n",
    "    # print(\"Mean accuracy:\", cv_scores.mean())\n",
    "    # with open(output_file_name, \"a\") as f: print('mean accuracy', cv_scores.mean() , file = f)\n",
    "\n",
    "\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed training time ', time_taken, file = f)\n",
    "    # joblib.dump(logreg_01, 'logreg_01.joblib')\n",
    "\n",
    "\n",
    "# if 1 == 1:\n",
    "    # logreg_01 = joblib.load('logreg_01.joblib')\n",
    "\n",
    "if 1 == 1:\n",
    "\n",
    "    #lR\n",
    "    start = time.time()\n",
    "    preds_lr = preds_logreg =logreg_00.predict(X_test)\n",
    "    preds_lr_prob = logreg_00.predict_proba(X_test)\n",
    "    end = time.time()\n",
    "    time_taken = end - start\n",
    "    with open(output_file_name, \"a\") as f: print('Elapsed prediction time ', time_taken, file = f)\n",
    "    with open(output_file_name, \"a\") as f: print('---------------------------------------------------------------------------------', file = f)\n",
    "\n",
    "#LR\n",
    "if 1 == 1:\n",
    "    pred_label = preds_logreg\n",
    "    name = 'lr'\n",
    "    metrics = confusion_metrics(name, pred_label, y_test)\n",
    "\n",
    "    Acc = metrics[0]\n",
    "    Precision = metrics[1]\n",
    "    Recall = metrics[2]\n",
    "    F1 = metrics[3]\n",
    "    BACC = metrics[4]\n",
    "    MCC = metrics[5]    \n",
    "\n",
    "    globals()[f\"{name}_acc_00\"] = Acc\n",
    "    globals()[f\"{name}_pre_00\"] = Precision\n",
    "    globals()[f\"{name}_rec_00\"] = Recall\n",
    "    globals()[f\"{name}_f1_00\"] = F1\n",
    "    globals()[f\"{name}_bacc_00\"] = BACC\n",
    "    globals()[f\"{name}_mcc_00\"] = MCC\n",
    "    \n",
    "    end = time.time()\n",
    "    time_taken = end - start_lr\n",
    "    globals()[f\"{name}_time_00\"] = time_taken"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bagging DT  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0       1      2      3      4      5        6\n",
      "0  12679.0     0.0    0.0    0.0    0.0    0.0      5.0\n",
      "1      0.0  5270.0    0.0    0.0    0.0    0.0      4.0\n",
      "2      0.0     0.0  417.0    0.0    0.0    0.0      2.0\n",
      "3      0.0     0.0    0.0  465.0    0.0    0.0      9.0\n",
      "4      0.0     0.0    0.0    0.0  448.0    0.0      1.0\n",
      "5      0.0     0.0    0.0    0.0    0.0  471.0      0.0\n",
      "6     13.0    11.0    2.0   18.0    0.0    0.0  37729.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.998870429584318\n",
      "Precision total:  0.9934708788688411\n",
      "Recall total:  0.9959563898301874\n",
      "F1 total:  0.9947010875602247\n",
      "BACC total:  0.9959563898301874\n",
      "MCC total:  0.997794534682812\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import BaggingClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.datasets import load_iris\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score\n",
    "start = time.time()\n",
    "# # Define the base classifier (Decision Tree in this case)\n",
    "base_classifier = DecisionTreeClassifier(random_state=42)\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test)\n",
    "\n",
    "# Evaluate accuracy\n",
    "# accuracy = accuracy_score(y_test, y_pred)\n",
    "# print(f'Accuracy: {accuracy}')\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "\n",
    "pred_label = y_pred\n",
    "name = 'bag_dt'\n",
    "metrics = confusion_metrics(name, pred_label, y_test)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "globals()[f\"{name}_acc_00\"] = Acc\n",
    "globals()[f\"{name}_pre_00\"] = Precision\n",
    "globals()[f\"{name}_rec_00\"] = Recall\n",
    "globals()[f\"{name}_f1_00\"] = F1\n",
    "globals()[f\"{name}_bacc_00\"] = BACC\n",
    "globals()[f\"{name}_mcc_00\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_00\"] = time_taken"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## bagging SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0       1     2        3    4    5    6\n",
      "0  11348.0    61.0   0.0   1275.0  0.0  0.0  0.0\n",
      "1      6.0  5218.0   0.0     50.0  0.0  0.0  0.0\n",
      "2      1.0     0.0   0.0    418.0  0.0  0.0  0.0\n",
      "3      0.0   114.0   0.0    360.0  0.0  0.0  0.0\n",
      "4      0.0     0.0   0.0    449.0  0.0  0.0  0.0\n",
      "5     71.0     0.0  70.0    330.0  0.0  0.0  0.0\n",
      "6    628.0  1060.0  11.0  36074.0  0.0  0.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9147782566383984\n",
      "Precision total:  0.3822950680895981\n",
      "Recall total:  0.4055818723346916\n",
      "F1 total:  0.3925233519825036\n",
      "BACC total:  0.4055818723346916\n",
      "MCC total:  0.8305570279186061\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "## bagging  with SVM\n",
    "from sklearn.ensemble import BaggingClassifier\n",
    "# from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "start = time.time()\n",
    "\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "\n",
    "# Instantiate the SGDClassifier with additional hyperparameters\n",
    "svm_01 = SGDClassifier(\n",
    "    loss='hinge',           # hinge loss for linear SVM\n",
    "    penalty='l2',           # L2 regularization to prevent overfitting\n",
    "    alpha=1e-4,             # Learning rate (small value for fine-grained updates)\n",
    "    max_iter=1000,          # Number of passes over the training data\n",
    "    random_state=42,        # Seed for reproducible results\n",
    "    learning_rate='optimal' # Automatically adjusts the learning rate based on the training data\n",
    ")\n",
    "\n",
    "# # Define the base classifier (Decision Tree in this case)\n",
    "base_classifier = svm_01\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test)\n",
    "\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_svm'\n",
    "pred_label = y_pred\n",
    "metrics = confusion_metrics(name, pred_label, y_test)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_00\"] = Acc\n",
    "globals()[f\"{name}_pre_00\"] = Precision\n",
    "globals()[f\"{name}_rec_00\"] = Recall\n",
    "globals()[f\"{name}_f1_00\"] = F1\n",
    "globals()[f\"{name}_bacc_00\"] = BACC\n",
    "globals()[f\"{name}_mcc_00\"] = MCC\n",
    "\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_00\"] = time_taken"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bagging MLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0       1      2      3      4      5        6\n",
      "0  12595.0     0.0    0.0    0.0    0.0    0.0     89.0\n",
      "1      3.0  5265.0    0.0    0.0    0.0    0.0      6.0\n",
      "2      1.0     0.0  415.0    0.0    0.0    0.0      3.0\n",
      "3      0.0     0.0    0.0  310.0    0.0    0.0    164.0\n",
      "4      0.0     0.0   27.0    0.0  420.0    0.0      2.0\n",
      "5      0.0     0.0    0.0    0.0    0.0  471.0      0.0\n",
      "6    163.0   526.0    1.0   20.0   95.0    4.0  36964.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9808146809398026\n",
      "Precision total:  0.9388990986768562\n",
      "Recall total:  0.935676187571112\n",
      "F1 total:  0.9326349460164448\n",
      "BACC total:  0.935676187571112\n",
      "MCC total:  0.9631026924168653\n"
     ]
    }
   ],
   "source": [
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.multioutput import MultiOutputClassifier\n",
    "import time\n",
    "start = time.time()\n",
    "# create MLPClassifier instance\n",
    "mlp_00 = MLPClassifier(hidden_layer_sizes=(100,), max_iter=200, random_state=1)\n",
    "\n",
    "base_classifier = mlp_00\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test)\n",
    "\n",
    "# Evaluate accuracy\n",
    "# accuracy = accuracy_score(y_test_00, y_pred)\n",
    "# print(f'Accuracy: {accuracy}')\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_mlp'\n",
    "pred_label = y_pred\n",
    "metrics = confusion_metrics(name, pred_label, y_test)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_00\"] = Acc\n",
    "globals()[f\"{name}_pre_00\"] = Precision\n",
    "globals()[f\"{name}_rec_00\"] = Recall\n",
    "globals()[f\"{name}_f1_00\"] = F1\n",
    "globals()[f\"{name}_bacc_00\"] = BACC\n",
    "globals()[f\"{name}_mcc_00\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_00\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### bagging KNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0       1      2      3      4      5        6\n",
      "0  12658.0     0.0    0.0    0.0    0.0    0.0     26.0\n",
      "1      5.0  5122.0    0.0    0.0    0.0    0.0    147.0\n",
      "2      0.0     0.0  417.0    0.0    0.0    0.0      2.0\n",
      "3      0.0     0.0    0.0  449.0    0.0    0.0     25.0\n",
      "4      0.0     0.0    1.0    0.0  447.0    0.0      1.0\n",
      "5      0.0     0.0    0.0    0.0    0.0  471.0      0.0\n",
      "6     41.0   346.0    3.0   59.0    8.0   12.0  37304.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9882524676769081\n",
      "Precision total:  0.9656675000439711\n",
      "Recall total:  0.9849632912942331\n",
      "F1 total:  0.9750829220826659\n",
      "BACC total:  0.9849632912942331\n",
      "MCC total:  0.9772137986500221\n"
     ]
    }
   ],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "knn_00=KNeighborsClassifier(n_neighbors = 5)\n",
    "start = time.time()\n",
    "base_classifier = knn_00\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test)\n",
    "\n",
    "# Evaluate accuracy\n",
    "# accuracy = accuracy_score(y_test_00, y_pred)\n",
    "# print(f'Accuracy: {accuracy}')\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_knn'\n",
    "\n",
    "pred_label = y_pred\n",
    "\n",
    "\n",
    "metrics = confusion_metrics(name, pred_label, y_test)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_00\"] = Acc\n",
    "globals()[f\"{name}_pre_00\"] = Precision\n",
    "globals()[f\"{name}_rec_00\"] = Recall\n",
    "globals()[f\"{name}_f1_00\"] = F1\n",
    "globals()[f\"{name}_bacc_00\"] = BACC\n",
    "globals()[f\"{name}_mcc_00\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_00\"] = time_taken"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "### bag LogRegression\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "Defining baggin Logistic Regression Model\n",
      "---------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "lbfgs failed to converge (status=2):\n",
      "ABNORMAL_TERMINATION_IN_LNSRCH.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "lbfgs failed to converge (status=2):\n",
      "ABNORMAL_TERMINATION_IN_LNSRCH.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "lbfgs failed to converge (status=2):\n",
      "ABNORMAL_TERMINATION_IN_LNSRCH.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "lbfgs failed to converge (status=2):\n",
      "ABNORMAL_TERMINATION_IN_LNSRCH.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "lbfgs failed to converge (status=2):\n",
      "ABNORMAL_TERMINATION_IN_LNSRCH.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "lbfgs failed to converge (status=2):\n",
      "ABNORMAL_TERMINATION_IN_LNSRCH.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "lbfgs failed to converge (status=2):\n",
      "ABNORMAL_TERMINATION_IN_LNSRCH.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "lbfgs failed to converge (status=2):\n",
      "ABNORMAL_TERMINATION_IN_LNSRCH.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0       1    2    3     4      5        6\n",
      "0  11638.0    61.0  2.0  0.0   0.0    3.0    980.0\n",
      "1      4.0  5225.0  0.0  0.0   0.0    0.0     45.0\n",
      "2      1.0     0.0  0.0  0.0   0.0    0.0    418.0\n",
      "3      0.0   114.0  0.0  2.0   0.0    0.0    358.0\n",
      "4      0.0     0.0  0.0  0.0   0.0    0.0    449.0\n",
      "5     30.0     0.0  0.0  0.0   0.0  398.0     43.0\n",
      "6    615.0   918.0  2.0  4.0  54.0   24.0  36156.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9283157236201863\n",
      "Precision total:  0.5691816514624338\n",
      "Recall total:  0.5306663871386478\n",
      "F1 total:  0.5255709115360113\n",
      "BACC total:  0.5306663871386478\n",
      "MCC total:  0.8585139143900585\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "start = time.time()\n",
    "#Logistic Regression\n",
    "print('---------------------------------------------------------------------------------')\n",
    "print('Defining baggin Logistic Regression Model')\n",
    "print('---------------------------------------------------------------------------------')\n",
    "logreg_00 = LogisticRegression()\n",
    "\n",
    "\n",
    "base_classifier = logreg_00\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test)\n",
    "\n",
    "# Evaluate accuracy\n",
    "# accuracy = accuracy_score(y_test_00, y_pred)\n",
    "# print(f'Accuracy: {accuracy}')\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_lr'\n",
    "\n",
    "pred_label = y_pred\n",
    "\n",
    "\n",
    "metrics = confusion_metrics(name, pred_label, y_test)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_00\"] = Acc\n",
    "globals()[f\"{name}_pre_00\"] = Precision\n",
    "globals()[f\"{name}_rec_00\"] = Recall\n",
    "globals()[f\"{name}_f1_00\"] = F1\n",
    "globals()[f\"{name}_bacc_00\"] = BACC\n",
    "globals()[f\"{name}_mcc_00\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_00\"] = time_taken"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bagging RF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0       1      2        3    4    5    6\n",
      "0  12078.0     0.0    0.0    606.0  0.0  0.0  0.0\n",
      "1      4.0  5223.0    0.0     47.0  0.0  0.0  0.0\n",
      "2      1.0     0.0    0.0    418.0  0.0  0.0  0.0\n",
      "3      0.0     0.0    0.0    474.0  0.0  0.0  0.0\n",
      "4      0.0     0.0    0.0    449.0  0.0  0.0  0.0\n",
      "5      0.0     0.0  321.0    150.0  0.0  0.0  0.0\n",
      "6    216.0    20.0    0.0  37537.0  0.0  0.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9585534547476714\n",
      "Precision total:  0.5605979361544021\n",
      "Recall total:  0.5168334296010132\n",
      "F1 total:  0.5342892371946626\n",
      "BACC total:  0.5168334296010132\n",
      "MCC total:  0.9177901422723888\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "rf = RandomForestClassifier(max_depth = 5,  n_estimators = 10, min_samples_split = 2, n_jobs = -1)\n",
    "\n",
    "base_classifier = rf\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_rf'\n",
    "\n",
    "pred_label = y_pred\n",
    "\n",
    "\n",
    "metrics = confusion_metrics(name, pred_label, y_test)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_00\"] = Acc\n",
    "globals()[f\"{name}_pre_00\"] = Precision\n",
    "globals()[f\"{name}_rec_00\"] = Recall\n",
    "globals()[f\"{name}_f1_00\"] = F1\n",
    "globals()[f\"{name}_bacc_00\"] = BACC\n",
    "globals()[f\"{name}_mcc_00\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_00\"] = time_taken\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bagging ADA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "        0      1     2        3    4    5    6\n",
      "0  7974.0    0.0   0.0   4710.0  0.0  0.0  0.0\n",
      "1     4.0   80.0   0.0   5190.0  0.0  0.0  0.0\n",
      "2     0.0  419.0   0.0      0.0  0.0  0.0  0.0\n",
      "3     0.0    0.0   0.0    474.0  0.0  0.0  0.0\n",
      "4     0.0    0.0   0.0    449.0  0.0  0.0  0.0\n",
      "5     0.0    0.0  95.0    376.0  0.0  0.0  0.0\n",
      "6    49.0  268.0   1.0  37455.0  0.0  0.0  0.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.7983977478103712\n",
      "Precision total:  0.47129835635444745\n",
      "Recall total:  0.4031351197104249\n",
      "F1 total:  0.38263462174576474\n",
      "BACC total:  0.4031351197104249\n",
      "MCC total:  0.5765360466696628\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "import time\n",
    "ada = AdaBoostClassifier(n_estimators=50, learning_rate=1.0)\n",
    "\n",
    "base_classifier = ada\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_ada'\n",
    "\n",
    "pred_label = y_pred\n",
    "\n",
    "\n",
    "metrics = confusion_metrics(name, pred_label, y_test)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_00\"] = Acc\n",
    "globals()[f\"{name}_pre_00\"] = Precision\n",
    "globals()[f\"{name}_rec_00\"] = Recall\n",
    "globals()[f\"{name}_f1_00\"] = F1\n",
    "globals()[f\"{name}_bacc_00\"] = BACC\n",
    "globals()[f\"{name}_mcc_00\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_00\"] = time_taken\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bagging LGBM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0       1      2      3      4      5        6\n",
      "0  12682.0     0.0    0.0    0.0    0.0    0.0      2.0\n",
      "1      0.0  5270.0    0.0    0.0    0.0    0.0      4.0\n",
      "2      0.0     0.0  418.0    0.0    0.0    0.0      1.0\n",
      "3      0.0     0.0    0.0  469.0    0.0    0.0      5.0\n",
      "4      0.0     0.0    0.0    0.0  449.0    0.0      0.0\n",
      "5      0.0     0.0    0.0    0.0    0.0  470.0      1.0\n",
      "6     10.0     9.0    0.0    8.0    0.0    0.0  37746.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9993048797441958\n",
      "Precision total:  0.9971987795024077\n",
      "Recall total:  0.9976158266900941\n",
      "F1 total:  0.9974054849350702\n",
      "BACC total:  0.9976158266900941\n",
      "MCC total:  0.9986424570887864\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "from lightgbm import LGBMClassifier\n",
    "lgbm = LGBMClassifier()\n",
    "\n",
    "\n",
    "base_classifier = lgbm\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_lgbm'\n",
    "\n",
    "pred_label = y_pred\n",
    "\n",
    "\n",
    "metrics = confusion_metrics(name, pred_label, y_test)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_00\"] = Acc\n",
    "globals()[f\"{name}_pre_00\"] = Precision\n",
    "globals()[f\"{name}_rec_00\"] = Recall\n",
    "globals()[f\"{name}_f1_00\"] = F1\n",
    "globals()[f\"{name}_bacc_00\"] = BACC\n",
    "globals()[f\"{name}_mcc_00\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_00\"] = time_taken\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bagging Catboost "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:\tlearn: 1.4413174\ttotal: 55.7ms\tremaining: 5.51s\n",
      "1:\tlearn: 1.1766511\ttotal: 104ms\tremaining: 5.11s\n",
      "2:\tlearn: 0.9973049\ttotal: 155ms\tremaining: 5.01s\n",
      "3:\tlearn: 0.8605123\ttotal: 205ms\tremaining: 4.92s\n",
      "4:\tlearn: 0.7545710\ttotal: 255ms\tremaining: 4.85s\n",
      "5:\tlearn: 0.6683602\ttotal: 305ms\tremaining: 4.78s\n",
      "6:\tlearn: 0.5941270\ttotal: 357ms\tremaining: 4.74s\n",
      "7:\tlearn: 0.5349887\ttotal: 406ms\tremaining: 4.67s\n",
      "8:\tlearn: 0.4828228\ttotal: 456ms\tremaining: 4.61s\n",
      "9:\tlearn: 0.4371718\ttotal: 506ms\tremaining: 4.55s\n",
      "10:\tlearn: 0.3964408\ttotal: 554ms\tremaining: 4.48s\n",
      "11:\tlearn: 0.3617605\ttotal: 604ms\tremaining: 4.43s\n",
      "12:\tlearn: 0.3314222\ttotal: 652ms\tremaining: 4.36s\n",
      "13:\tlearn: 0.3039553\ttotal: 700ms\tremaining: 4.3s\n",
      "14:\tlearn: 0.2795702\ttotal: 751ms\tremaining: 4.25s\n",
      "15:\tlearn: 0.2577371\ttotal: 800ms\tremaining: 4.2s\n",
      "16:\tlearn: 0.2391662\ttotal: 850ms\tremaining: 4.15s\n",
      "17:\tlearn: 0.2218009\ttotal: 896ms\tremaining: 4.08s\n",
      "18:\tlearn: 0.2059186\ttotal: 944ms\tremaining: 4.02s\n",
      "19:\tlearn: 0.1907388\ttotal: 990ms\tremaining: 3.96s\n",
      "20:\tlearn: 0.1774106\ttotal: 1.04s\tremaining: 3.9s\n",
      "21:\tlearn: 0.1666024\ttotal: 1.09s\tremaining: 3.86s\n",
      "22:\tlearn: 0.1552909\ttotal: 1.13s\tremaining: 3.79s\n",
      "23:\tlearn: 0.1457272\ttotal: 1.18s\tremaining: 3.73s\n",
      "24:\tlearn: 0.1373214\ttotal: 1.23s\tremaining: 3.68s\n",
      "25:\tlearn: 0.1289760\ttotal: 1.27s\tremaining: 3.62s\n",
      "26:\tlearn: 0.1219153\ttotal: 1.32s\tremaining: 3.56s\n",
      "27:\tlearn: 0.1148909\ttotal: 1.37s\tremaining: 3.51s\n",
      "28:\tlearn: 0.1077653\ttotal: 1.41s\tremaining: 3.46s\n",
      "29:\tlearn: 0.1006076\ttotal: 1.46s\tremaining: 3.42s\n",
      "30:\tlearn: 0.0953910\ttotal: 1.51s\tremaining: 3.36s\n",
      "31:\tlearn: 0.0902475\ttotal: 1.56s\tremaining: 3.31s\n",
      "32:\tlearn: 0.0859738\ttotal: 1.6s\tremaining: 3.26s\n",
      "33:\tlearn: 0.0816598\ttotal: 1.65s\tremaining: 3.21s\n",
      "34:\tlearn: 0.0776040\ttotal: 1.7s\tremaining: 3.16s\n",
      "35:\tlearn: 0.0741161\ttotal: 1.75s\tremaining: 3.11s\n",
      "36:\tlearn: 0.0705142\ttotal: 1.79s\tremaining: 3.05s\n",
      "37:\tlearn: 0.0673311\ttotal: 1.84s\tremaining: 3s\n",
      "38:\tlearn: 0.0642229\ttotal: 1.89s\tremaining: 2.95s\n",
      "39:\tlearn: 0.0618554\ttotal: 1.93s\tremaining: 2.9s\n",
      "40:\tlearn: 0.0592878\ttotal: 1.98s\tremaining: 2.85s\n",
      "41:\tlearn: 0.0568819\ttotal: 2.03s\tremaining: 2.8s\n",
      "42:\tlearn: 0.0548512\ttotal: 2.07s\tremaining: 2.74s\n",
      "43:\tlearn: 0.0533374\ttotal: 2.11s\tremaining: 2.68s\n",
      "44:\tlearn: 0.0522153\ttotal: 2.15s\tremaining: 2.63s\n",
      "45:\tlearn: 0.0512537\ttotal: 2.19s\tremaining: 2.57s\n",
      "46:\tlearn: 0.0496597\ttotal: 2.24s\tremaining: 2.53s\n",
      "47:\tlearn: 0.0472277\ttotal: 2.29s\tremaining: 2.48s\n",
      "48:\tlearn: 0.0453949\ttotal: 2.34s\tremaining: 2.44s\n",
      "49:\tlearn: 0.0438422\ttotal: 2.39s\tremaining: 2.39s\n",
      "50:\tlearn: 0.0431791\ttotal: 2.44s\tremaining: 2.34s\n",
      "51:\tlearn: 0.0426269\ttotal: 2.47s\tremaining: 2.28s\n",
      "52:\tlearn: 0.0413987\ttotal: 2.52s\tremaining: 2.23s\n",
      "53:\tlearn: 0.0401812\ttotal: 2.56s\tremaining: 2.18s\n",
      "54:\tlearn: 0.0390725\ttotal: 2.61s\tremaining: 2.13s\n",
      "55:\tlearn: 0.0383313\ttotal: 2.65s\tremaining: 2.08s\n",
      "56:\tlearn: 0.0373787\ttotal: 2.69s\tremaining: 2.03s\n",
      "57:\tlearn: 0.0365443\ttotal: 2.74s\tremaining: 1.98s\n",
      "58:\tlearn: 0.0356230\ttotal: 2.78s\tremaining: 1.93s\n",
      "59:\tlearn: 0.0347115\ttotal: 2.83s\tremaining: 1.89s\n",
      "60:\tlearn: 0.0339072\ttotal: 2.87s\tremaining: 1.83s\n",
      "61:\tlearn: 0.0331192\ttotal: 2.92s\tremaining: 1.79s\n",
      "62:\tlearn: 0.0324024\ttotal: 2.96s\tremaining: 1.74s\n",
      "63:\tlearn: 0.0316482\ttotal: 3.01s\tremaining: 1.69s\n",
      "64:\tlearn: 0.0306997\ttotal: 3.06s\tremaining: 1.65s\n",
      "65:\tlearn: 0.0300560\ttotal: 3.11s\tremaining: 1.6s\n",
      "66:\tlearn: 0.0295784\ttotal: 3.15s\tremaining: 1.55s\n",
      "67:\tlearn: 0.0289457\ttotal: 3.2s\tremaining: 1.51s\n",
      "68:\tlearn: 0.0287925\ttotal: 3.24s\tremaining: 1.46s\n",
      "69:\tlearn: 0.0281212\ttotal: 3.29s\tremaining: 1.41s\n",
      "70:\tlearn: 0.0279295\ttotal: 3.33s\tremaining: 1.36s\n",
      "71:\tlearn: 0.0273726\ttotal: 3.37s\tremaining: 1.31s\n",
      "72:\tlearn: 0.0271135\ttotal: 3.42s\tremaining: 1.26s\n",
      "73:\tlearn: 0.0265917\ttotal: 3.47s\tremaining: 1.22s\n",
      "74:\tlearn: 0.0262258\ttotal: 3.51s\tremaining: 1.17s\n",
      "75:\tlearn: 0.0257538\ttotal: 3.55s\tremaining: 1.12s\n",
      "76:\tlearn: 0.0253393\ttotal: 3.6s\tremaining: 1.07s\n",
      "77:\tlearn: 0.0248022\ttotal: 3.64s\tremaining: 1.03s\n",
      "78:\tlearn: 0.0244989\ttotal: 3.68s\tremaining: 979ms\n",
      "79:\tlearn: 0.0238883\ttotal: 3.73s\tremaining: 933ms\n",
      "80:\tlearn: 0.0234754\ttotal: 3.78s\tremaining: 886ms\n",
      "81:\tlearn: 0.0232776\ttotal: 3.82s\tremaining: 839ms\n",
      "82:\tlearn: 0.0228442\ttotal: 3.87s\tremaining: 794ms\n",
      "83:\tlearn: 0.0223878\ttotal: 3.92s\tremaining: 747ms\n",
      "84:\tlearn: 0.0219736\ttotal: 3.97s\tremaining: 701ms\n",
      "85:\tlearn: 0.0218744\ttotal: 4.01s\tremaining: 653ms\n",
      "86:\tlearn: 0.0215526\ttotal: 4.06s\tremaining: 607ms\n",
      "87:\tlearn: 0.0212776\ttotal: 4.1s\tremaining: 560ms\n",
      "88:\tlearn: 0.0209541\ttotal: 4.15s\tremaining: 513ms\n",
      "89:\tlearn: 0.0206921\ttotal: 4.2s\tremaining: 466ms\n",
      "90:\tlearn: 0.0204781\ttotal: 4.24s\tremaining: 419ms\n",
      "91:\tlearn: 0.0202629\ttotal: 4.28s\tremaining: 373ms\n",
      "92:\tlearn: 0.0201438\ttotal: 4.33s\tremaining: 326ms\n",
      "93:\tlearn: 0.0200076\ttotal: 4.37s\tremaining: 279ms\n",
      "94:\tlearn: 0.0196867\ttotal: 4.41s\tremaining: 232ms\n",
      "95:\tlearn: 0.0194506\ttotal: 4.46s\tremaining: 186ms\n",
      "96:\tlearn: 0.0192346\ttotal: 4.5s\tremaining: 139ms\n",
      "97:\tlearn: 0.0189791\ttotal: 4.55s\tremaining: 92.8ms\n",
      "98:\tlearn: 0.0187681\ttotal: 4.59s\tremaining: 46.4ms\n",
      "99:\tlearn: 0.0186202\ttotal: 4.64s\tremaining: 0us\n",
      "0:\tlearn: 1.4405391\ttotal: 61.4ms\tremaining: 6.08s\n",
      "1:\tlearn: 1.1722786\ttotal: 112ms\tremaining: 5.49s\n",
      "2:\tlearn: 0.9921526\ttotal: 161ms\tremaining: 5.22s\n",
      "3:\tlearn: 0.8551755\ttotal: 212ms\tremaining: 5.08s\n",
      "4:\tlearn: 0.7497359\ttotal: 260ms\tremaining: 4.95s\n",
      "5:\tlearn: 0.6635671\ttotal: 311ms\tremaining: 4.87s\n",
      "6:\tlearn: 0.5902659\ttotal: 363ms\tremaining: 4.82s\n",
      "7:\tlearn: 0.5312601\ttotal: 412ms\tremaining: 4.73s\n",
      "8:\tlearn: 0.4778239\ttotal: 463ms\tremaining: 4.68s\n",
      "9:\tlearn: 0.4356574\ttotal: 512ms\tremaining: 4.61s\n",
      "10:\tlearn: 0.3951993\ttotal: 560ms\tremaining: 4.53s\n",
      "11:\tlearn: 0.3598339\ttotal: 611ms\tremaining: 4.48s\n",
      "12:\tlearn: 0.3305921\ttotal: 661ms\tremaining: 4.42s\n",
      "13:\tlearn: 0.3036051\ttotal: 710ms\tremaining: 4.36s\n",
      "14:\tlearn: 0.2794769\ttotal: 761ms\tremaining: 4.31s\n",
      "15:\tlearn: 0.2580842\ttotal: 808ms\tremaining: 4.24s\n",
      "16:\tlearn: 0.2385959\ttotal: 857ms\tremaining: 4.18s\n",
      "17:\tlearn: 0.2207440\ttotal: 905ms\tremaining: 4.12s\n",
      "18:\tlearn: 0.2051281\ttotal: 952ms\tremaining: 4.06s\n",
      "19:\tlearn: 0.1909303\ttotal: 996ms\tremaining: 3.98s\n",
      "20:\tlearn: 0.1771608\ttotal: 1.04s\tremaining: 3.91s\n",
      "21:\tlearn: 0.1662709\ttotal: 1.09s\tremaining: 3.87s\n",
      "22:\tlearn: 0.1554682\ttotal: 1.14s\tremaining: 3.81s\n",
      "23:\tlearn: 0.1455270\ttotal: 1.19s\tremaining: 3.75s\n",
      "24:\tlearn: 0.1374250\ttotal: 1.23s\tremaining: 3.7s\n",
      "25:\tlearn: 0.1296285\ttotal: 1.28s\tremaining: 3.63s\n",
      "26:\tlearn: 0.1224277\ttotal: 1.32s\tremaining: 3.58s\n",
      "27:\tlearn: 0.1154775\ttotal: 1.37s\tremaining: 3.52s\n",
      "28:\tlearn: 0.1093479\ttotal: 1.41s\tremaining: 3.46s\n",
      "29:\tlearn: 0.1032471\ttotal: 1.46s\tremaining: 3.4s\n",
      "30:\tlearn: 0.0980363\ttotal: 1.51s\tremaining: 3.35s\n",
      "31:\tlearn: 0.0927514\ttotal: 1.55s\tremaining: 3.3s\n",
      "32:\tlearn: 0.0888200\ttotal: 1.6s\tremaining: 3.24s\n",
      "33:\tlearn: 0.0841290\ttotal: 1.64s\tremaining: 3.19s\n",
      "34:\tlearn: 0.0796688\ttotal: 1.69s\tremaining: 3.14s\n",
      "35:\tlearn: 0.0764441\ttotal: 1.74s\tremaining: 3.09s\n",
      "36:\tlearn: 0.0727274\ttotal: 1.78s\tremaining: 3.04s\n",
      "37:\tlearn: 0.0696337\ttotal: 1.83s\tremaining: 2.98s\n",
      "38:\tlearn: 0.0671267\ttotal: 1.87s\tremaining: 2.93s\n",
      "39:\tlearn: 0.0638734\ttotal: 1.92s\tremaining: 2.88s\n",
      "40:\tlearn: 0.0611914\ttotal: 1.97s\tremaining: 2.83s\n",
      "41:\tlearn: 0.0586399\ttotal: 2.01s\tremaining: 2.78s\n",
      "42:\tlearn: 0.0565500\ttotal: 2.06s\tremaining: 2.72s\n",
      "43:\tlearn: 0.0551668\ttotal: 2.1s\tremaining: 2.67s\n",
      "44:\tlearn: 0.0537730\ttotal: 2.14s\tremaining: 2.62s\n",
      "45:\tlearn: 0.0523896\ttotal: 2.18s\tremaining: 2.56s\n",
      "46:\tlearn: 0.0505685\ttotal: 2.23s\tremaining: 2.51s\n",
      "47:\tlearn: 0.0482346\ttotal: 2.28s\tremaining: 2.47s\n",
      "48:\tlearn: 0.0461051\ttotal: 2.33s\tremaining: 2.42s\n",
      "49:\tlearn: 0.0447177\ttotal: 2.37s\tremaining: 2.37s\n",
      "50:\tlearn: 0.0433308\ttotal: 2.42s\tremaining: 2.32s\n",
      "51:\tlearn: 0.0425316\ttotal: 2.46s\tremaining: 2.27s\n",
      "52:\tlearn: 0.0412448\ttotal: 2.51s\tremaining: 2.22s\n",
      "53:\tlearn: 0.0399419\ttotal: 2.55s\tremaining: 2.17s\n",
      "54:\tlearn: 0.0389614\ttotal: 2.6s\tremaining: 2.12s\n",
      "55:\tlearn: 0.0380933\ttotal: 2.64s\tremaining: 2.08s\n",
      "56:\tlearn: 0.0374333\ttotal: 2.68s\tremaining: 2.02s\n",
      "57:\tlearn: 0.0362749\ttotal: 2.73s\tremaining: 1.98s\n",
      "58:\tlearn: 0.0353463\ttotal: 2.78s\tremaining: 1.93s\n",
      "59:\tlearn: 0.0346309\ttotal: 2.82s\tremaining: 1.88s\n",
      "60:\tlearn: 0.0339115\ttotal: 2.87s\tremaining: 1.83s\n",
      "61:\tlearn: 0.0329360\ttotal: 2.91s\tremaining: 1.78s\n",
      "62:\tlearn: 0.0321990\ttotal: 2.95s\tremaining: 1.73s\n",
      "63:\tlearn: 0.0314825\ttotal: 3s\tremaining: 1.69s\n",
      "64:\tlearn: 0.0308143\ttotal: 3.04s\tremaining: 1.64s\n",
      "65:\tlearn: 0.0301686\ttotal: 3.09s\tremaining: 1.59s\n",
      "66:\tlearn: 0.0297636\ttotal: 3.13s\tremaining: 1.54s\n",
      "67:\tlearn: 0.0291905\ttotal: 3.18s\tremaining: 1.5s\n",
      "68:\tlearn: 0.0285677\ttotal: 3.23s\tremaining: 1.45s\n",
      "69:\tlearn: 0.0282006\ttotal: 3.27s\tremaining: 1.4s\n",
      "70:\tlearn: 0.0278381\ttotal: 3.32s\tremaining: 1.35s\n",
      "71:\tlearn: 0.0273827\ttotal: 3.36s\tremaining: 1.31s\n",
      "72:\tlearn: 0.0266685\ttotal: 3.41s\tremaining: 1.26s\n",
      "73:\tlearn: 0.0261702\ttotal: 3.45s\tremaining: 1.21s\n",
      "74:\tlearn: 0.0257666\ttotal: 3.5s\tremaining: 1.17s\n",
      "75:\tlearn: 0.0253085\ttotal: 3.54s\tremaining: 1.12s\n",
      "76:\tlearn: 0.0250805\ttotal: 3.58s\tremaining: 1.07s\n",
      "77:\tlearn: 0.0245141\ttotal: 3.63s\tremaining: 1.02s\n",
      "78:\tlearn: 0.0241109\ttotal: 3.67s\tremaining: 977ms\n",
      "79:\tlearn: 0.0236932\ttotal: 3.72s\tremaining: 930ms\n",
      "80:\tlearn: 0.0233989\ttotal: 3.76s\tremaining: 883ms\n",
      "81:\tlearn: 0.0231155\ttotal: 3.81s\tremaining: 836ms\n",
      "82:\tlearn: 0.0226703\ttotal: 3.86s\tremaining: 790ms\n",
      "83:\tlearn: 0.0224086\ttotal: 3.9s\tremaining: 743ms\n",
      "84:\tlearn: 0.0221080\ttotal: 3.94s\tremaining: 696ms\n",
      "85:\tlearn: 0.0218426\ttotal: 3.99s\tremaining: 649ms\n",
      "86:\tlearn: 0.0215479\ttotal: 4.04s\tremaining: 603ms\n",
      "87:\tlearn: 0.0212960\ttotal: 4.08s\tremaining: 556ms\n",
      "88:\tlearn: 0.0210047\ttotal: 4.13s\tremaining: 510ms\n",
      "89:\tlearn: 0.0206626\ttotal: 4.17s\tremaining: 464ms\n",
      "90:\tlearn: 0.0203201\ttotal: 4.22s\tremaining: 418ms\n",
      "91:\tlearn: 0.0200278\ttotal: 4.27s\tremaining: 371ms\n",
      "92:\tlearn: 0.0198615\ttotal: 4.31s\tremaining: 325ms\n",
      "93:\tlearn: 0.0195124\ttotal: 4.36s\tremaining: 278ms\n",
      "94:\tlearn: 0.0191931\ttotal: 4.41s\tremaining: 232ms\n",
      "95:\tlearn: 0.0189698\ttotal: 4.45s\tremaining: 186ms\n",
      "96:\tlearn: 0.0187251\ttotal: 4.5s\tremaining: 139ms\n",
      "97:\tlearn: 0.0184383\ttotal: 4.54s\tremaining: 92.7ms\n",
      "98:\tlearn: 0.0182171\ttotal: 4.58s\tremaining: 46.3ms\n",
      "99:\tlearn: 0.0181302\ttotal: 4.62s\tremaining: 0us\n",
      "0:\tlearn: 1.4402275\ttotal: 57.6ms\tremaining: 5.7s\n",
      "1:\tlearn: 1.1753298\ttotal: 106ms\tremaining: 5.21s\n",
      "2:\tlearn: 0.9957803\ttotal: 157ms\tremaining: 5.06s\n",
      "3:\tlearn: 0.8586045\ttotal: 206ms\tremaining: 4.95s\n",
      "4:\tlearn: 0.7524814\ttotal: 257ms\tremaining: 4.88s\n",
      "5:\tlearn: 0.6660518\ttotal: 307ms\tremaining: 4.82s\n",
      "6:\tlearn: 0.5921710\ttotal: 360ms\tremaining: 4.78s\n",
      "7:\tlearn: 0.5328351\ttotal: 409ms\tremaining: 4.7s\n",
      "8:\tlearn: 0.4790430\ttotal: 460ms\tremaining: 4.65s\n",
      "9:\tlearn: 0.4368369\ttotal: 510ms\tremaining: 4.59s\n",
      "10:\tlearn: 0.3957498\ttotal: 559ms\tremaining: 4.53s\n",
      "11:\tlearn: 0.3582954\ttotal: 609ms\tremaining: 4.47s\n",
      "12:\tlearn: 0.3291301\ttotal: 657ms\tremaining: 4.4s\n",
      "13:\tlearn: 0.3015930\ttotal: 705ms\tremaining: 4.33s\n",
      "14:\tlearn: 0.2766511\ttotal: 751ms\tremaining: 4.25s\n",
      "15:\tlearn: 0.2553384\ttotal: 799ms\tremaining: 4.2s\n",
      "16:\tlearn: 0.2360839\ttotal: 849ms\tremaining: 4.15s\n",
      "17:\tlearn: 0.2178161\ttotal: 895ms\tremaining: 4.08s\n",
      "18:\tlearn: 0.2033778\ttotal: 943ms\tremaining: 4.02s\n",
      "19:\tlearn: 0.1901016\ttotal: 989ms\tremaining: 3.95s\n",
      "20:\tlearn: 0.1775295\ttotal: 1.04s\tremaining: 3.9s\n",
      "21:\tlearn: 0.1666632\ttotal: 1.09s\tremaining: 3.85s\n",
      "22:\tlearn: 0.1553654\ttotal: 1.14s\tremaining: 3.81s\n",
      "23:\tlearn: 0.1453635\ttotal: 1.19s\tremaining: 3.77s\n",
      "24:\tlearn: 0.1362441\ttotal: 1.24s\tremaining: 3.71s\n",
      "25:\tlearn: 0.1284514\ttotal: 1.28s\tremaining: 3.65s\n",
      "26:\tlearn: 0.1207833\ttotal: 1.33s\tremaining: 3.6s\n",
      "27:\tlearn: 0.1133854\ttotal: 1.38s\tremaining: 3.55s\n",
      "28:\tlearn: 0.1066872\ttotal: 1.43s\tremaining: 3.49s\n",
      "29:\tlearn: 0.1008831\ttotal: 1.48s\tremaining: 3.45s\n",
      "30:\tlearn: 0.0954330\ttotal: 1.52s\tremaining: 3.4s\n",
      "31:\tlearn: 0.0905674\ttotal: 1.57s\tremaining: 3.34s\n",
      "32:\tlearn: 0.0858559\ttotal: 1.62s\tremaining: 3.29s\n",
      "33:\tlearn: 0.0813146\ttotal: 1.67s\tremaining: 3.24s\n",
      "34:\tlearn: 0.0774122\ttotal: 1.72s\tremaining: 3.19s\n",
      "35:\tlearn: 0.0744042\ttotal: 1.76s\tremaining: 3.13s\n",
      "36:\tlearn: 0.0709223\ttotal: 1.81s\tremaining: 3.08s\n",
      "37:\tlearn: 0.0683464\ttotal: 1.85s\tremaining: 3.02s\n",
      "38:\tlearn: 0.0650420\ttotal: 1.9s\tremaining: 2.97s\n",
      "39:\tlearn: 0.0631675\ttotal: 1.95s\tremaining: 2.92s\n",
      "40:\tlearn: 0.0604879\ttotal: 1.99s\tremaining: 2.87s\n",
      "41:\tlearn: 0.0579902\ttotal: 2.04s\tremaining: 2.82s\n",
      "42:\tlearn: 0.0551751\ttotal: 2.09s\tremaining: 2.77s\n",
      "43:\tlearn: 0.0534781\ttotal: 2.14s\tremaining: 2.72s\n",
      "44:\tlearn: 0.0522125\ttotal: 2.18s\tremaining: 2.67s\n",
      "45:\tlearn: 0.0508641\ttotal: 2.23s\tremaining: 2.62s\n",
      "46:\tlearn: 0.0490063\ttotal: 2.27s\tremaining: 2.56s\n",
      "47:\tlearn: 0.0468160\ttotal: 2.32s\tremaining: 2.52s\n",
      "48:\tlearn: 0.0455675\ttotal: 2.37s\tremaining: 2.47s\n",
      "49:\tlearn: 0.0443119\ttotal: 2.41s\tremaining: 2.41s\n",
      "50:\tlearn: 0.0432868\ttotal: 2.45s\tremaining: 2.36s\n",
      "51:\tlearn: 0.0420609\ttotal: 2.5s\tremaining: 2.31s\n",
      "52:\tlearn: 0.0409472\ttotal: 2.54s\tremaining: 2.26s\n",
      "53:\tlearn: 0.0399645\ttotal: 2.59s\tremaining: 2.21s\n",
      "54:\tlearn: 0.0387903\ttotal: 2.64s\tremaining: 2.16s\n",
      "55:\tlearn: 0.0380179\ttotal: 2.68s\tremaining: 2.11s\n",
      "56:\tlearn: 0.0368623\ttotal: 2.73s\tremaining: 2.06s\n",
      "57:\tlearn: 0.0362300\ttotal: 2.77s\tremaining: 2.01s\n",
      "58:\tlearn: 0.0352278\ttotal: 2.82s\tremaining: 1.96s\n",
      "59:\tlearn: 0.0345762\ttotal: 2.87s\tremaining: 1.91s\n",
      "60:\tlearn: 0.0338347\ttotal: 2.91s\tremaining: 1.86s\n",
      "61:\tlearn: 0.0330034\ttotal: 2.96s\tremaining: 1.81s\n",
      "62:\tlearn: 0.0322541\ttotal: 3s\tremaining: 1.76s\n",
      "63:\tlearn: 0.0315548\ttotal: 3.05s\tremaining: 1.72s\n",
      "64:\tlearn: 0.0309432\ttotal: 3.1s\tremaining: 1.67s\n",
      "65:\tlearn: 0.0303278\ttotal: 3.14s\tremaining: 1.62s\n",
      "66:\tlearn: 0.0297850\ttotal: 3.18s\tremaining: 1.57s\n",
      "67:\tlearn: 0.0288653\ttotal: 3.24s\tremaining: 1.52s\n",
      "68:\tlearn: 0.0282780\ttotal: 3.28s\tremaining: 1.47s\n",
      "69:\tlearn: 0.0275934\ttotal: 3.33s\tremaining: 1.43s\n",
      "70:\tlearn: 0.0270899\ttotal: 3.38s\tremaining: 1.38s\n",
      "71:\tlearn: 0.0266349\ttotal: 3.42s\tremaining: 1.33s\n",
      "72:\tlearn: 0.0264462\ttotal: 3.46s\tremaining: 1.28s\n",
      "73:\tlearn: 0.0259621\ttotal: 3.51s\tremaining: 1.23s\n",
      "74:\tlearn: 0.0253725\ttotal: 3.55s\tremaining: 1.18s\n",
      "75:\tlearn: 0.0249116\ttotal: 3.6s\tremaining: 1.14s\n",
      "76:\tlearn: 0.0246245\ttotal: 3.64s\tremaining: 1.09s\n",
      "77:\tlearn: 0.0240189\ttotal: 3.69s\tremaining: 1.04s\n",
      "78:\tlearn: 0.0236029\ttotal: 3.74s\tremaining: 993ms\n",
      "79:\tlearn: 0.0231721\ttotal: 3.79s\tremaining: 947ms\n",
      "80:\tlearn: 0.0228540\ttotal: 3.83s\tremaining: 899ms\n",
      "81:\tlearn: 0.0225283\ttotal: 3.88s\tremaining: 852ms\n",
      "82:\tlearn: 0.0221781\ttotal: 3.93s\tremaining: 805ms\n",
      "83:\tlearn: 0.0217222\ttotal: 3.98s\tremaining: 758ms\n",
      "84:\tlearn: 0.0213895\ttotal: 4.02s\tremaining: 710ms\n",
      "85:\tlearn: 0.0211217\ttotal: 4.07s\tremaining: 662ms\n",
      "86:\tlearn: 0.0208387\ttotal: 4.11s\tremaining: 614ms\n",
      "87:\tlearn: 0.0205457\ttotal: 4.16s\tremaining: 567ms\n",
      "88:\tlearn: 0.0202675\ttotal: 4.2s\tremaining: 520ms\n",
      "89:\tlearn: 0.0200073\ttotal: 4.25s\tremaining: 472ms\n",
      "90:\tlearn: 0.0196526\ttotal: 4.3s\tremaining: 425ms\n",
      "91:\tlearn: 0.0194103\ttotal: 4.35s\tremaining: 378ms\n",
      "92:\tlearn: 0.0191062\ttotal: 4.39s\tremaining: 331ms\n",
      "93:\tlearn: 0.0190372\ttotal: 4.43s\tremaining: 283ms\n",
      "94:\tlearn: 0.0188531\ttotal: 4.48s\tremaining: 236ms\n",
      "95:\tlearn: 0.0185695\ttotal: 4.52s\tremaining: 189ms\n",
      "96:\tlearn: 0.0184009\ttotal: 4.57s\tremaining: 141ms\n",
      "97:\tlearn: 0.0181307\ttotal: 4.61s\tremaining: 94.2ms\n",
      "98:\tlearn: 0.0178493\ttotal: 4.66s\tremaining: 47.1ms\n",
      "99:\tlearn: 0.0176832\ttotal: 4.7s\tremaining: 0us\n",
      "0:\tlearn: 1.4414277\ttotal: 55.5ms\tremaining: 5.49s\n",
      "1:\tlearn: 1.1766700\ttotal: 106ms\tremaining: 5.18s\n",
      "2:\tlearn: 0.9973922\ttotal: 157ms\tremaining: 5.07s\n",
      "3:\tlearn: 0.8603526\ttotal: 208ms\tremaining: 5s\n",
      "4:\tlearn: 0.7545630\ttotal: 260ms\tremaining: 4.93s\n",
      "5:\tlearn: 0.6682009\ttotal: 311ms\tremaining: 4.88s\n",
      "6:\tlearn: 0.5939481\ttotal: 365ms\tremaining: 4.84s\n",
      "7:\tlearn: 0.5346693\ttotal: 415ms\tremaining: 4.77s\n",
      "8:\tlearn: 0.4808714\ttotal: 468ms\tremaining: 4.73s\n",
      "9:\tlearn: 0.4385752\ttotal: 520ms\tremaining: 4.68s\n",
      "10:\tlearn: 0.4001521\ttotal: 571ms\tremaining: 4.62s\n",
      "11:\tlearn: 0.3612176\ttotal: 620ms\tremaining: 4.55s\n",
      "12:\tlearn: 0.3312368\ttotal: 670ms\tremaining: 4.49s\n",
      "13:\tlearn: 0.3045388\ttotal: 722ms\tremaining: 4.44s\n",
      "14:\tlearn: 0.2814979\ttotal: 774ms\tremaining: 4.39s\n",
      "15:\tlearn: 0.2596505\ttotal: 824ms\tremaining: 4.33s\n",
      "16:\tlearn: 0.2398421\ttotal: 874ms\tremaining: 4.27s\n",
      "17:\tlearn: 0.2230964\ttotal: 921ms\tremaining: 4.2s\n",
      "18:\tlearn: 0.2084356\ttotal: 969ms\tremaining: 4.13s\n",
      "19:\tlearn: 0.1933128\ttotal: 1.02s\tremaining: 4.06s\n",
      "20:\tlearn: 0.1800627\ttotal: 1.06s\tremaining: 4s\n",
      "21:\tlearn: 0.1684745\ttotal: 1.12s\tremaining: 3.96s\n",
      "22:\tlearn: 0.1570268\ttotal: 1.16s\tremaining: 3.89s\n",
      "23:\tlearn: 0.1469905\ttotal: 1.21s\tremaining: 3.83s\n",
      "24:\tlearn: 0.1379571\ttotal: 1.26s\tremaining: 3.77s\n",
      "25:\tlearn: 0.1308743\ttotal: 1.3s\tremaining: 3.71s\n",
      "26:\tlearn: 0.1232283\ttotal: 1.35s\tremaining: 3.66s\n",
      "27:\tlearn: 0.1156535\ttotal: 1.4s\tremaining: 3.61s\n",
      "28:\tlearn: 0.1092044\ttotal: 1.45s\tremaining: 3.56s\n",
      "29:\tlearn: 0.1032002\ttotal: 1.5s\tremaining: 3.5s\n",
      "30:\tlearn: 0.0974163\ttotal: 1.55s\tremaining: 3.45s\n",
      "31:\tlearn: 0.0929117\ttotal: 1.6s\tremaining: 3.4s\n",
      "32:\tlearn: 0.0882818\ttotal: 1.65s\tremaining: 3.35s\n",
      "33:\tlearn: 0.0836747\ttotal: 1.7s\tremaining: 3.29s\n",
      "34:\tlearn: 0.0794150\ttotal: 1.75s\tremaining: 3.25s\n",
      "35:\tlearn: 0.0771069\ttotal: 1.79s\tremaining: 3.18s\n",
      "36:\tlearn: 0.0734619\ttotal: 1.84s\tremaining: 3.13s\n",
      "37:\tlearn: 0.0695171\ttotal: 1.89s\tremaining: 3.08s\n",
      "38:\tlearn: 0.0669697\ttotal: 1.94s\tremaining: 3.03s\n",
      "39:\tlearn: 0.0639870\ttotal: 1.98s\tremaining: 2.97s\n",
      "40:\tlearn: 0.0618628\ttotal: 2.03s\tremaining: 2.92s\n",
      "41:\tlearn: 0.0590649\ttotal: 2.08s\tremaining: 2.87s\n",
      "42:\tlearn: 0.0573039\ttotal: 2.12s\tremaining: 2.82s\n",
      "43:\tlearn: 0.0556364\ttotal: 2.17s\tremaining: 2.76s\n",
      "44:\tlearn: 0.0535850\ttotal: 2.21s\tremaining: 2.71s\n",
      "45:\tlearn: 0.0517576\ttotal: 2.26s\tremaining: 2.65s\n",
      "46:\tlearn: 0.0498147\ttotal: 2.31s\tremaining: 2.61s\n",
      "47:\tlearn: 0.0475104\ttotal: 2.36s\tremaining: 2.56s\n",
      "48:\tlearn: 0.0460697\ttotal: 2.41s\tremaining: 2.51s\n",
      "49:\tlearn: 0.0446593\ttotal: 2.46s\tremaining: 2.46s\n",
      "50:\tlearn: 0.0431670\ttotal: 2.51s\tremaining: 2.41s\n",
      "51:\tlearn: 0.0421617\ttotal: 2.55s\tremaining: 2.36s\n",
      "52:\tlearn: 0.0414364\ttotal: 2.6s\tremaining: 2.3s\n",
      "53:\tlearn: 0.0402878\ttotal: 2.65s\tremaining: 2.26s\n",
      "54:\tlearn: 0.0390053\ttotal: 2.7s\tremaining: 2.21s\n",
      "55:\tlearn: 0.0381157\ttotal: 2.75s\tremaining: 2.16s\n",
      "56:\tlearn: 0.0373636\ttotal: 2.79s\tremaining: 2.1s\n",
      "57:\tlearn: 0.0366953\ttotal: 2.83s\tremaining: 2.05s\n",
      "58:\tlearn: 0.0353932\ttotal: 2.89s\tremaining: 2s\n",
      "59:\tlearn: 0.0345782\ttotal: 2.93s\tremaining: 1.95s\n",
      "60:\tlearn: 0.0336546\ttotal: 2.98s\tremaining: 1.9s\n",
      "61:\tlearn: 0.0329608\ttotal: 3.02s\tremaining: 1.85s\n",
      "62:\tlearn: 0.0322330\ttotal: 3.07s\tremaining: 1.8s\n",
      "63:\tlearn: 0.0314746\ttotal: 3.12s\tremaining: 1.75s\n",
      "64:\tlearn: 0.0307964\ttotal: 3.16s\tremaining: 1.7s\n",
      "65:\tlearn: 0.0303572\ttotal: 3.21s\tremaining: 1.65s\n",
      "66:\tlearn: 0.0297193\ttotal: 3.25s\tremaining: 1.6s\n",
      "67:\tlearn: 0.0290976\ttotal: 3.3s\tremaining: 1.55s\n",
      "68:\tlearn: 0.0285316\ttotal: 3.35s\tremaining: 1.5s\n",
      "69:\tlearn: 0.0278625\ttotal: 3.4s\tremaining: 1.46s\n",
      "70:\tlearn: 0.0273154\ttotal: 3.45s\tremaining: 1.41s\n",
      "71:\tlearn: 0.0269710\ttotal: 3.5s\tremaining: 1.36s\n",
      "72:\tlearn: 0.0265941\ttotal: 3.54s\tremaining: 1.31s\n",
      "73:\tlearn: 0.0261260\ttotal: 3.59s\tremaining: 1.26s\n",
      "74:\tlearn: 0.0255877\ttotal: 3.64s\tremaining: 1.21s\n",
      "75:\tlearn: 0.0251749\ttotal: 3.68s\tremaining: 1.16s\n",
      "76:\tlearn: 0.0249013\ttotal: 3.73s\tremaining: 1.11s\n",
      "77:\tlearn: 0.0245666\ttotal: 3.78s\tremaining: 1.07s\n",
      "78:\tlearn: 0.0241925\ttotal: 3.83s\tremaining: 1.02s\n",
      "79:\tlearn: 0.0236518\ttotal: 3.88s\tremaining: 970ms\n",
      "80:\tlearn: 0.0232402\ttotal: 3.93s\tremaining: 921ms\n",
      "81:\tlearn: 0.0230537\ttotal: 3.97s\tremaining: 872ms\n",
      "82:\tlearn: 0.0227352\ttotal: 4.02s\tremaining: 823ms\n",
      "83:\tlearn: 0.0224053\ttotal: 4.07s\tremaining: 775ms\n",
      "84:\tlearn: 0.0221295\ttotal: 4.11s\tremaining: 726ms\n",
      "85:\tlearn: 0.0217581\ttotal: 4.16s\tremaining: 678ms\n",
      "86:\tlearn: 0.0214696\ttotal: 4.21s\tremaining: 629ms\n",
      "87:\tlearn: 0.0211157\ttotal: 4.26s\tremaining: 581ms\n",
      "88:\tlearn: 0.0208270\ttotal: 4.3s\tremaining: 532ms\n",
      "89:\tlearn: 0.0205637\ttotal: 4.35s\tremaining: 484ms\n",
      "90:\tlearn: 0.0203383\ttotal: 4.4s\tremaining: 435ms\n",
      "91:\tlearn: 0.0201674\ttotal: 4.44s\tremaining: 386ms\n",
      "92:\tlearn: 0.0196666\ttotal: 4.51s\tremaining: 339ms\n",
      "93:\tlearn: 0.0193034\ttotal: 4.56s\tremaining: 291ms\n",
      "94:\tlearn: 0.0191692\ttotal: 4.6s\tremaining: 242ms\n",
      "95:\tlearn: 0.0190482\ttotal: 4.64s\tremaining: 193ms\n",
      "96:\tlearn: 0.0187879\ttotal: 4.69s\tremaining: 145ms\n",
      "97:\tlearn: 0.0186291\ttotal: 4.75s\tremaining: 96.9ms\n",
      "98:\tlearn: 0.0182551\ttotal: 4.79s\tremaining: 48.4ms\n",
      "99:\tlearn: 0.0181335\ttotal: 4.84s\tremaining: 0us\n",
      "0:\tlearn: 1.4414890\ttotal: 65.3ms\tremaining: 6.46s\n",
      "1:\tlearn: 1.1769202\ttotal: 116ms\tremaining: 5.7s\n",
      "2:\tlearn: 0.9975972\ttotal: 170ms\tremaining: 5.48s\n",
      "3:\tlearn: 0.8605560\ttotal: 222ms\tremaining: 5.33s\n",
      "4:\tlearn: 0.7547375\ttotal: 274ms\tremaining: 5.21s\n",
      "5:\tlearn: 0.6685794\ttotal: 327ms\tremaining: 5.12s\n",
      "6:\tlearn: 0.5945223\ttotal: 381ms\tremaining: 5.06s\n",
      "7:\tlearn: 0.5353127\ttotal: 432ms\tremaining: 4.96s\n",
      "8:\tlearn: 0.4815519\ttotal: 486ms\tremaining: 4.92s\n",
      "9:\tlearn: 0.4380136\ttotal: 538ms\tremaining: 4.84s\n",
      "10:\tlearn: 0.3966391\ttotal: 589ms\tremaining: 4.77s\n",
      "11:\tlearn: 0.3629037\ttotal: 641ms\tremaining: 4.7s\n",
      "12:\tlearn: 0.3318730\ttotal: 692ms\tremaining: 4.63s\n",
      "13:\tlearn: 0.3058228\ttotal: 743ms\tremaining: 4.56s\n",
      "14:\tlearn: 0.2814015\ttotal: 797ms\tremaining: 4.51s\n",
      "15:\tlearn: 0.2610378\ttotal: 849ms\tremaining: 4.46s\n",
      "16:\tlearn: 0.2424220\ttotal: 902ms\tremaining: 4.4s\n",
      "17:\tlearn: 0.2246626\ttotal: 950ms\tremaining: 4.33s\n",
      "18:\tlearn: 0.2104956\ttotal: 999ms\tremaining: 4.26s\n",
      "19:\tlearn: 0.1946250\ttotal: 1.05s\tremaining: 4.18s\n",
      "20:\tlearn: 0.1816505\ttotal: 1.09s\tremaining: 4.11s\n",
      "21:\tlearn: 0.1697333\ttotal: 1.15s\tremaining: 4.07s\n",
      "22:\tlearn: 0.1580311\ttotal: 1.19s\tremaining: 4s\n",
      "23:\tlearn: 0.1495760\ttotal: 1.24s\tremaining: 3.93s\n",
      "24:\tlearn: 0.1410943\ttotal: 1.29s\tremaining: 3.87s\n",
      "25:\tlearn: 0.1327023\ttotal: 1.34s\tremaining: 3.81s\n",
      "26:\tlearn: 0.1254263\ttotal: 1.39s\tremaining: 3.75s\n",
      "27:\tlearn: 0.1190159\ttotal: 1.44s\tremaining: 3.69s\n",
      "28:\tlearn: 0.1121983\ttotal: 1.49s\tremaining: 3.64s\n",
      "29:\tlearn: 0.1058983\ttotal: 1.54s\tremaining: 3.59s\n",
      "30:\tlearn: 0.1008666\ttotal: 1.59s\tremaining: 3.53s\n",
      "31:\tlearn: 0.0959234\ttotal: 1.64s\tremaining: 3.48s\n",
      "32:\tlearn: 0.0908063\ttotal: 1.69s\tremaining: 3.43s\n",
      "33:\tlearn: 0.0861134\ttotal: 1.74s\tremaining: 3.37s\n",
      "34:\tlearn: 0.0815848\ttotal: 1.79s\tremaining: 3.32s\n",
      "35:\tlearn: 0.0777645\ttotal: 1.84s\tremaining: 3.27s\n",
      "36:\tlearn: 0.0730694\ttotal: 1.89s\tremaining: 3.22s\n",
      "37:\tlearn: 0.0699911\ttotal: 1.94s\tremaining: 3.17s\n",
      "38:\tlearn: 0.0668872\ttotal: 1.99s\tremaining: 3.11s\n",
      "39:\tlearn: 0.0643683\ttotal: 2.04s\tremaining: 3.06s\n",
      "40:\tlearn: 0.0607705\ttotal: 2.09s\tremaining: 3.01s\n",
      "41:\tlearn: 0.0586596\ttotal: 2.14s\tremaining: 2.96s\n",
      "42:\tlearn: 0.0566911\ttotal: 2.18s\tremaining: 2.9s\n",
      "43:\tlearn: 0.0551364\ttotal: 2.23s\tremaining: 2.83s\n",
      "44:\tlearn: 0.0534782\ttotal: 2.28s\tremaining: 2.78s\n",
      "45:\tlearn: 0.0523464\ttotal: 2.32s\tremaining: 2.72s\n",
      "46:\tlearn: 0.0505232\ttotal: 2.37s\tremaining: 2.67s\n",
      "47:\tlearn: 0.0486748\ttotal: 2.42s\tremaining: 2.62s\n",
      "48:\tlearn: 0.0472622\ttotal: 2.47s\tremaining: 2.57s\n",
      "49:\tlearn: 0.0460571\ttotal: 2.51s\tremaining: 2.51s\n",
      "50:\tlearn: 0.0447750\ttotal: 2.56s\tremaining: 2.46s\n",
      "51:\tlearn: 0.0440222\ttotal: 2.61s\tremaining: 2.41s\n",
      "52:\tlearn: 0.0428297\ttotal: 2.65s\tremaining: 2.35s\n",
      "53:\tlearn: 0.0411157\ttotal: 2.71s\tremaining: 2.31s\n",
      "54:\tlearn: 0.0399294\ttotal: 2.75s\tremaining: 2.25s\n",
      "55:\tlearn: 0.0390045\ttotal: 2.81s\tremaining: 2.21s\n",
      "56:\tlearn: 0.0380949\ttotal: 2.86s\tremaining: 2.16s\n",
      "57:\tlearn: 0.0371804\ttotal: 2.9s\tremaining: 2.1s\n",
      "58:\tlearn: 0.0363295\ttotal: 2.96s\tremaining: 2.05s\n",
      "59:\tlearn: 0.0353566\ttotal: 3s\tremaining: 2s\n",
      "60:\tlearn: 0.0345829\ttotal: 3.05s\tremaining: 1.95s\n",
      "61:\tlearn: 0.0337859\ttotal: 3.1s\tremaining: 1.9s\n",
      "62:\tlearn: 0.0328460\ttotal: 3.15s\tremaining: 1.85s\n",
      "63:\tlearn: 0.0321899\ttotal: 3.2s\tremaining: 1.8s\n",
      "64:\tlearn: 0.0314683\ttotal: 3.25s\tremaining: 1.75s\n",
      "65:\tlearn: 0.0309588\ttotal: 3.3s\tremaining: 1.7s\n",
      "66:\tlearn: 0.0303021\ttotal: 3.35s\tremaining: 1.65s\n",
      "67:\tlearn: 0.0293543\ttotal: 3.4s\tremaining: 1.6s\n",
      "68:\tlearn: 0.0287818\ttotal: 3.45s\tremaining: 1.55s\n",
      "69:\tlearn: 0.0283230\ttotal: 3.5s\tremaining: 1.5s\n",
      "70:\tlearn: 0.0276607\ttotal: 3.56s\tremaining: 1.45s\n",
      "71:\tlearn: 0.0270589\ttotal: 3.6s\tremaining: 1.4s\n",
      "72:\tlearn: 0.0265936\ttotal: 3.65s\tremaining: 1.35s\n",
      "73:\tlearn: 0.0260479\ttotal: 3.7s\tremaining: 1.3s\n",
      "74:\tlearn: 0.0256715\ttotal: 3.75s\tremaining: 1.25s\n",
      "75:\tlearn: 0.0251922\ttotal: 3.79s\tremaining: 1.2s\n",
      "76:\tlearn: 0.0247857\ttotal: 3.84s\tremaining: 1.15s\n",
      "77:\tlearn: 0.0242025\ttotal: 3.9s\tremaining: 1.1s\n",
      "78:\tlearn: 0.0239420\ttotal: 3.94s\tremaining: 1.05s\n",
      "79:\tlearn: 0.0235115\ttotal: 3.99s\tremaining: 997ms\n",
      "80:\tlearn: 0.0230157\ttotal: 4.04s\tremaining: 947ms\n",
      "81:\tlearn: 0.0226281\ttotal: 4.08s\tremaining: 897ms\n",
      "82:\tlearn: 0.0222934\ttotal: 4.14s\tremaining: 848ms\n",
      "83:\tlearn: 0.0219208\ttotal: 4.18s\tremaining: 797ms\n",
      "84:\tlearn: 0.0217455\ttotal: 4.23s\tremaining: 747ms\n",
      "85:\tlearn: 0.0214611\ttotal: 4.28s\tremaining: 697ms\n",
      "86:\tlearn: 0.0211983\ttotal: 4.33s\tremaining: 647ms\n",
      "87:\tlearn: 0.0209223\ttotal: 4.38s\tremaining: 597ms\n",
      "88:\tlearn: 0.0207379\ttotal: 4.43s\tremaining: 547ms\n",
      "89:\tlearn: 0.0204485\ttotal: 4.48s\tremaining: 498ms\n",
      "90:\tlearn: 0.0201342\ttotal: 4.53s\tremaining: 448ms\n",
      "91:\tlearn: 0.0198198\ttotal: 4.58s\tremaining: 399ms\n",
      "92:\tlearn: 0.0194642\ttotal: 4.63s\tremaining: 349ms\n",
      "93:\tlearn: 0.0192679\ttotal: 4.68s\tremaining: 299ms\n",
      "94:\tlearn: 0.0190281\ttotal: 4.73s\tremaining: 249ms\n",
      "95:\tlearn: 0.0187678\ttotal: 4.79s\tremaining: 199ms\n",
      "96:\tlearn: 0.0185837\ttotal: 4.83s\tremaining: 149ms\n",
      "97:\tlearn: 0.0182872\ttotal: 4.88s\tremaining: 99.6ms\n",
      "98:\tlearn: 0.0181254\ttotal: 4.95s\tremaining: 50ms\n",
      "99:\tlearn: 0.0178989\ttotal: 5s\tremaining: 0us\n",
      "0:\tlearn: 1.4420502\ttotal: 57.4ms\tremaining: 5.68s\n",
      "1:\tlearn: 1.1779269\ttotal: 108ms\tremaining: 5.28s\n",
      "2:\tlearn: 0.9974165\ttotal: 160ms\tremaining: 5.18s\n",
      "3:\tlearn: 0.8608377\ttotal: 213ms\tremaining: 5.11s\n",
      "4:\tlearn: 0.7553088\ttotal: 265ms\tremaining: 5.04s\n",
      "5:\tlearn: 0.6693946\ttotal: 317ms\tremaining: 4.97s\n",
      "6:\tlearn: 0.5952233\ttotal: 372ms\tremaining: 4.94s\n",
      "7:\tlearn: 0.5361844\ttotal: 424ms\tremaining: 4.88s\n",
      "8:\tlearn: 0.4839050\ttotal: 478ms\tremaining: 4.83s\n",
      "9:\tlearn: 0.4396792\ttotal: 530ms\tremaining: 4.77s\n",
      "10:\tlearn: 0.3996353\ttotal: 582ms\tremaining: 4.7s\n",
      "11:\tlearn: 0.3659284\ttotal: 633ms\tremaining: 4.64s\n",
      "12:\tlearn: 0.3346351\ttotal: 685ms\tremaining: 4.59s\n",
      "13:\tlearn: 0.3072179\ttotal: 737ms\tremaining: 4.52s\n",
      "14:\tlearn: 0.2826191\ttotal: 789ms\tremaining: 4.47s\n",
      "15:\tlearn: 0.2609346\ttotal: 840ms\tremaining: 4.41s\n",
      "16:\tlearn: 0.2413232\ttotal: 892ms\tremaining: 4.36s\n",
      "17:\tlearn: 0.2233832\ttotal: 942ms\tremaining: 4.29s\n",
      "18:\tlearn: 0.2093656\ttotal: 991ms\tremaining: 4.22s\n",
      "19:\tlearn: 0.1933489\ttotal: 1.04s\tremaining: 4.16s\n",
      "20:\tlearn: 0.1803928\ttotal: 1.09s\tremaining: 4.09s\n",
      "21:\tlearn: 0.1684707\ttotal: 1.13s\tremaining: 4.02s\n",
      "22:\tlearn: 0.1570205\ttotal: 1.18s\tremaining: 3.96s\n",
      "23:\tlearn: 0.1485822\ttotal: 1.23s\tremaining: 3.89s\n",
      "24:\tlearn: 0.1393047\ttotal: 1.28s\tremaining: 3.83s\n",
      "25:\tlearn: 0.1322375\ttotal: 1.33s\tremaining: 3.78s\n",
      "26:\tlearn: 0.1244810\ttotal: 1.38s\tremaining: 3.73s\n",
      "27:\tlearn: 0.1165729\ttotal: 1.43s\tremaining: 3.67s\n",
      "28:\tlearn: 0.1097489\ttotal: 1.48s\tremaining: 3.62s\n",
      "29:\tlearn: 0.1038676\ttotal: 1.53s\tremaining: 3.57s\n",
      "30:\tlearn: 0.0985147\ttotal: 1.58s\tremaining: 3.52s\n",
      "31:\tlearn: 0.0931942\ttotal: 1.63s\tremaining: 3.47s\n",
      "32:\tlearn: 0.0890973\ttotal: 1.68s\tremaining: 3.4s\n",
      "33:\tlearn: 0.0852509\ttotal: 1.72s\tremaining: 3.35s\n",
      "34:\tlearn: 0.0814302\ttotal: 1.77s\tremaining: 3.29s\n",
      "35:\tlearn: 0.0783608\ttotal: 1.82s\tremaining: 3.24s\n",
      "36:\tlearn: 0.0745891\ttotal: 1.87s\tremaining: 3.18s\n",
      "37:\tlearn: 0.0704576\ttotal: 1.92s\tremaining: 3.13s\n",
      "38:\tlearn: 0.0668252\ttotal: 1.97s\tremaining: 3.08s\n",
      "39:\tlearn: 0.0630780\ttotal: 2.03s\tremaining: 3.04s\n",
      "40:\tlearn: 0.0606427\ttotal: 2.08s\tremaining: 2.99s\n",
      "41:\tlearn: 0.0587791\ttotal: 2.12s\tremaining: 2.93s\n",
      "42:\tlearn: 0.0568357\ttotal: 2.17s\tremaining: 2.88s\n",
      "43:\tlearn: 0.0552163\ttotal: 2.21s\tremaining: 2.82s\n",
      "44:\tlearn: 0.0534229\ttotal: 2.26s\tremaining: 2.77s\n",
      "45:\tlearn: 0.0519125\ttotal: 2.31s\tremaining: 2.71s\n",
      "46:\tlearn: 0.0502241\ttotal: 2.35s\tremaining: 2.66s\n",
      "47:\tlearn: 0.0481276\ttotal: 2.41s\tremaining: 2.61s\n",
      "48:\tlearn: 0.0469201\ttotal: 2.45s\tremaining: 2.55s\n",
      "49:\tlearn: 0.0457543\ttotal: 2.5s\tremaining: 2.5s\n",
      "50:\tlearn: 0.0448910\ttotal: 2.54s\tremaining: 2.44s\n",
      "51:\tlearn: 0.0442160\ttotal: 2.58s\tremaining: 2.38s\n",
      "52:\tlearn: 0.0429081\ttotal: 2.63s\tremaining: 2.33s\n",
      "53:\tlearn: 0.0414480\ttotal: 2.68s\tremaining: 2.29s\n",
      "54:\tlearn: 0.0402836\ttotal: 2.73s\tremaining: 2.23s\n",
      "55:\tlearn: 0.0391043\ttotal: 2.78s\tremaining: 2.18s\n",
      "56:\tlearn: 0.0378874\ttotal: 2.83s\tremaining: 2.13s\n",
      "57:\tlearn: 0.0370674\ttotal: 2.87s\tremaining: 2.08s\n",
      "58:\tlearn: 0.0362299\ttotal: 2.92s\tremaining: 2.03s\n",
      "59:\tlearn: 0.0349904\ttotal: 2.97s\tremaining: 1.98s\n",
      "60:\tlearn: 0.0341008\ttotal: 3.02s\tremaining: 1.93s\n",
      "61:\tlearn: 0.0333898\ttotal: 3.07s\tremaining: 1.88s\n",
      "62:\tlearn: 0.0327860\ttotal: 3.12s\tremaining: 1.83s\n",
      "63:\tlearn: 0.0320315\ttotal: 3.17s\tremaining: 1.78s\n",
      "64:\tlearn: 0.0313120\ttotal: 3.22s\tremaining: 1.73s\n",
      "65:\tlearn: 0.0308756\ttotal: 3.27s\tremaining: 1.68s\n",
      "66:\tlearn: 0.0300353\ttotal: 3.32s\tremaining: 1.63s\n",
      "67:\tlearn: 0.0294933\ttotal: 3.36s\tremaining: 1.58s\n",
      "68:\tlearn: 0.0286715\ttotal: 3.41s\tremaining: 1.53s\n",
      "69:\tlearn: 0.0279376\ttotal: 3.47s\tremaining: 1.49s\n",
      "70:\tlearn: 0.0273449\ttotal: 3.51s\tremaining: 1.43s\n",
      "71:\tlearn: 0.0268485\ttotal: 3.56s\tremaining: 1.38s\n",
      "72:\tlearn: 0.0263833\ttotal: 3.61s\tremaining: 1.33s\n",
      "73:\tlearn: 0.0258764\ttotal: 3.65s\tremaining: 1.28s\n",
      "74:\tlearn: 0.0254649\ttotal: 3.7s\tremaining: 1.23s\n",
      "75:\tlearn: 0.0250170\ttotal: 3.75s\tremaining: 1.18s\n",
      "76:\tlearn: 0.0245146\ttotal: 3.8s\tremaining: 1.14s\n",
      "77:\tlearn: 0.0241524\ttotal: 3.85s\tremaining: 1.08s\n",
      "78:\tlearn: 0.0236767\ttotal: 3.9s\tremaining: 1.04s\n",
      "79:\tlearn: 0.0234094\ttotal: 3.95s\tremaining: 987ms\n",
      "80:\tlearn: 0.0230304\ttotal: 4s\tremaining: 937ms\n",
      "81:\tlearn: 0.0227677\ttotal: 4.04s\tremaining: 888ms\n",
      "82:\tlearn: 0.0222927\ttotal: 4.09s\tremaining: 839ms\n",
      "83:\tlearn: 0.0221247\ttotal: 4.14s\tremaining: 789ms\n",
      "84:\tlearn: 0.0218343\ttotal: 4.19s\tremaining: 739ms\n",
      "85:\tlearn: 0.0215237\ttotal: 4.24s\tremaining: 690ms\n",
      "86:\tlearn: 0.0212607\ttotal: 4.29s\tremaining: 640ms\n",
      "87:\tlearn: 0.0209844\ttotal: 4.33s\tremaining: 591ms\n",
      "88:\tlearn: 0.0207007\ttotal: 4.38s\tremaining: 542ms\n",
      "89:\tlearn: 0.0205876\ttotal: 4.43s\tremaining: 492ms\n",
      "90:\tlearn: 0.0202484\ttotal: 4.48s\tremaining: 443ms\n",
      "91:\tlearn: 0.0200732\ttotal: 4.53s\tremaining: 393ms\n",
      "92:\tlearn: 0.0198770\ttotal: 4.57s\tremaining: 344ms\n",
      "93:\tlearn: 0.0196385\ttotal: 4.62s\tremaining: 295ms\n",
      "94:\tlearn: 0.0194544\ttotal: 4.67s\tremaining: 246ms\n",
      "95:\tlearn: 0.0191477\ttotal: 4.72s\tremaining: 197ms\n",
      "96:\tlearn: 0.0189406\ttotal: 4.76s\tremaining: 147ms\n",
      "97:\tlearn: 0.0187509\ttotal: 4.81s\tremaining: 98.2ms\n",
      "98:\tlearn: 0.0184487\ttotal: 4.86s\tremaining: 49.1ms\n",
      "99:\tlearn: 0.0183095\ttotal: 4.91s\tremaining: 0us\n",
      "0:\tlearn: 1.4417068\ttotal: 61.1ms\tremaining: 6.04s\n",
      "1:\tlearn: 1.1735489\ttotal: 115ms\tremaining: 5.66s\n",
      "2:\tlearn: 0.9930700\ttotal: 168ms\tremaining: 5.45s\n",
      "3:\tlearn: 0.8565791\ttotal: 221ms\tremaining: 5.31s\n",
      "4:\tlearn: 0.7512717\ttotal: 274ms\tremaining: 5.21s\n",
      "5:\tlearn: 0.6655268\ttotal: 327ms\tremaining: 5.12s\n",
      "6:\tlearn: 0.5919357\ttotal: 383ms\tremaining: 5.09s\n",
      "7:\tlearn: 0.5327772\ttotal: 434ms\tremaining: 5s\n",
      "8:\tlearn: 0.4793896\ttotal: 489ms\tremaining: 4.94s\n",
      "9:\tlearn: 0.4373037\ttotal: 542ms\tremaining: 4.88s\n",
      "10:\tlearn: 0.3988391\ttotal: 596ms\tremaining: 4.82s\n",
      "11:\tlearn: 0.3599516\ttotal: 647ms\tremaining: 4.74s\n",
      "12:\tlearn: 0.3296196\ttotal: 697ms\tremaining: 4.66s\n",
      "13:\tlearn: 0.3023657\ttotal: 749ms\tremaining: 4.6s\n",
      "14:\tlearn: 0.2798304\ttotal: 802ms\tremaining: 4.55s\n",
      "15:\tlearn: 0.2577815\ttotal: 855ms\tremaining: 4.49s\n",
      "16:\tlearn: 0.2389913\ttotal: 908ms\tremaining: 4.43s\n",
      "17:\tlearn: 0.2213578\ttotal: 957ms\tremaining: 4.36s\n",
      "18:\tlearn: 0.2059985\ttotal: 1.01s\tremaining: 4.29s\n",
      "19:\tlearn: 0.1912275\ttotal: 1.06s\tremaining: 4.23s\n",
      "20:\tlearn: 0.1786906\ttotal: 1.11s\tremaining: 4.17s\n",
      "21:\tlearn: 0.1673542\ttotal: 1.16s\tremaining: 4.12s\n",
      "22:\tlearn: 0.1561427\ttotal: 1.21s\tremaining: 4.05s\n",
      "23:\tlearn: 0.1462895\ttotal: 1.26s\tremaining: 3.98s\n",
      "24:\tlearn: 0.1373652\ttotal: 1.31s\tremaining: 3.92s\n",
      "25:\tlearn: 0.1294891\ttotal: 1.35s\tremaining: 3.85s\n",
      "26:\tlearn: 0.1213858\ttotal: 1.41s\tremaining: 3.81s\n",
      "27:\tlearn: 0.1150460\ttotal: 1.46s\tremaining: 3.75s\n",
      "28:\tlearn: 0.1083703\ttotal: 1.51s\tremaining: 3.7s\n",
      "29:\tlearn: 0.1023000\ttotal: 1.56s\tremaining: 3.64s\n",
      "30:\tlearn: 0.0970909\ttotal: 1.61s\tremaining: 3.58s\n",
      "31:\tlearn: 0.0927415\ttotal: 1.66s\tremaining: 3.52s\n",
      "32:\tlearn: 0.0878104\ttotal: 1.71s\tremaining: 3.47s\n",
      "33:\tlearn: 0.0834438\ttotal: 1.76s\tremaining: 3.42s\n",
      "34:\tlearn: 0.0796565\ttotal: 1.81s\tremaining: 3.37s\n",
      "35:\tlearn: 0.0774208\ttotal: 1.86s\tremaining: 3.3s\n",
      "36:\tlearn: 0.0737733\ttotal: 1.91s\tremaining: 3.25s\n",
      "37:\tlearn: 0.0698911\ttotal: 1.96s\tremaining: 3.2s\n",
      "38:\tlearn: 0.0667989\ttotal: 2.01s\tremaining: 3.14s\n",
      "39:\tlearn: 0.0648094\ttotal: 2.05s\tremaining: 3.08s\n",
      "40:\tlearn: 0.0614555\ttotal: 2.1s\tremaining: 3.03s\n",
      "41:\tlearn: 0.0589455\ttotal: 2.15s\tremaining: 2.97s\n",
      "42:\tlearn: 0.0566934\ttotal: 2.2s\tremaining: 2.92s\n",
      "43:\tlearn: 0.0550999\ttotal: 2.25s\tremaining: 2.86s\n",
      "44:\tlearn: 0.0537088\ttotal: 2.29s\tremaining: 2.8s\n",
      "45:\tlearn: 0.0521892\ttotal: 2.34s\tremaining: 2.74s\n",
      "46:\tlearn: 0.0504477\ttotal: 2.39s\tremaining: 2.69s\n",
      "47:\tlearn: 0.0482236\ttotal: 2.44s\tremaining: 2.65s\n",
      "48:\tlearn: 0.0461109\ttotal: 2.5s\tremaining: 2.6s\n",
      "49:\tlearn: 0.0447144\ttotal: 2.55s\tremaining: 2.55s\n",
      "50:\tlearn: 0.0437020\ttotal: 2.6s\tremaining: 2.5s\n",
      "51:\tlearn: 0.0419427\ttotal: 2.65s\tremaining: 2.45s\n",
      "52:\tlearn: 0.0408253\ttotal: 2.7s\tremaining: 2.39s\n",
      "53:\tlearn: 0.0396991\ttotal: 2.75s\tremaining: 2.34s\n",
      "54:\tlearn: 0.0385678\ttotal: 2.8s\tremaining: 2.29s\n",
      "55:\tlearn: 0.0375538\ttotal: 2.85s\tremaining: 2.24s\n",
      "56:\tlearn: 0.0365380\ttotal: 2.9s\tremaining: 2.19s\n",
      "57:\tlearn: 0.0352320\ttotal: 2.95s\tremaining: 2.14s\n",
      "58:\tlearn: 0.0341900\ttotal: 3s\tremaining: 2.09s\n",
      "59:\tlearn: 0.0332936\ttotal: 3.06s\tremaining: 2.04s\n",
      "60:\tlearn: 0.0325407\ttotal: 3.1s\tremaining: 1.98s\n",
      "61:\tlearn: 0.0317487\ttotal: 3.15s\tremaining: 1.93s\n",
      "62:\tlearn: 0.0311332\ttotal: 3.2s\tremaining: 1.88s\n",
      "63:\tlearn: 0.0304468\ttotal: 3.25s\tremaining: 1.83s\n",
      "64:\tlearn: 0.0300923\ttotal: 3.29s\tremaining: 1.77s\n",
      "65:\tlearn: 0.0294811\ttotal: 3.34s\tremaining: 1.72s\n",
      "66:\tlearn: 0.0288807\ttotal: 3.39s\tremaining: 1.67s\n",
      "67:\tlearn: 0.0281411\ttotal: 3.44s\tremaining: 1.62s\n",
      "68:\tlearn: 0.0277446\ttotal: 3.49s\tremaining: 1.57s\n",
      "69:\tlearn: 0.0272928\ttotal: 3.54s\tremaining: 1.52s\n",
      "70:\tlearn: 0.0266819\ttotal: 3.6s\tremaining: 1.47s\n",
      "71:\tlearn: 0.0260077\ttotal: 3.65s\tremaining: 1.42s\n",
      "72:\tlearn: 0.0257099\ttotal: 3.69s\tremaining: 1.37s\n",
      "73:\tlearn: 0.0252945\ttotal: 3.74s\tremaining: 1.31s\n",
      "74:\tlearn: 0.0247862\ttotal: 3.79s\tremaining: 1.26s\n",
      "75:\tlearn: 0.0245969\ttotal: 3.84s\tremaining: 1.21s\n",
      "76:\tlearn: 0.0241441\ttotal: 3.89s\tremaining: 1.16s\n",
      "77:\tlearn: 0.0237081\ttotal: 3.93s\tremaining: 1.11s\n",
      "78:\tlearn: 0.0233350\ttotal: 3.98s\tremaining: 1.06s\n",
      "79:\tlearn: 0.0230326\ttotal: 4.04s\tremaining: 1.01s\n",
      "80:\tlearn: 0.0226121\ttotal: 4.08s\tremaining: 958ms\n",
      "81:\tlearn: 0.0221399\ttotal: 4.14s\tremaining: 908ms\n",
      "82:\tlearn: 0.0218775\ttotal: 4.18s\tremaining: 857ms\n",
      "83:\tlearn: 0.0215088\ttotal: 4.24s\tremaining: 807ms\n",
      "84:\tlearn: 0.0213245\ttotal: 4.28s\tremaining: 756ms\n",
      "85:\tlearn: 0.0210761\ttotal: 4.33s\tremaining: 705ms\n",
      "86:\tlearn: 0.0207446\ttotal: 4.38s\tremaining: 654ms\n",
      "87:\tlearn: 0.0203977\ttotal: 4.43s\tremaining: 604ms\n",
      "88:\tlearn: 0.0200190\ttotal: 4.48s\tremaining: 554ms\n",
      "89:\tlearn: 0.0197344\ttotal: 4.53s\tremaining: 503ms\n",
      "90:\tlearn: 0.0194122\ttotal: 4.58s\tremaining: 453ms\n",
      "91:\tlearn: 0.0192461\ttotal: 4.63s\tremaining: 403ms\n",
      "92:\tlearn: 0.0189380\ttotal: 4.68s\tremaining: 352ms\n",
      "93:\tlearn: 0.0187572\ttotal: 4.73s\tremaining: 302ms\n",
      "94:\tlearn: 0.0186513\ttotal: 4.78s\tremaining: 251ms\n",
      "95:\tlearn: 0.0184646\ttotal: 4.82s\tremaining: 201ms\n",
      "96:\tlearn: 0.0182895\ttotal: 4.87s\tremaining: 151ms\n",
      "97:\tlearn: 0.0180882\ttotal: 4.92s\tremaining: 100ms\n",
      "98:\tlearn: 0.0178944\ttotal: 4.97s\tremaining: 50.2ms\n",
      "99:\tlearn: 0.0176447\ttotal: 5.01s\tremaining: 0us\n",
      "0:\tlearn: 1.4409429\ttotal: 64.9ms\tremaining: 6.43s\n",
      "1:\tlearn: 1.1738526\ttotal: 118ms\tremaining: 5.76s\n",
      "2:\tlearn: 0.9950740\ttotal: 170ms\tremaining: 5.49s\n",
      "3:\tlearn: 0.8580911\ttotal: 223ms\tremaining: 5.34s\n",
      "4:\tlearn: 0.7523114\ttotal: 275ms\tremaining: 5.23s\n",
      "5:\tlearn: 0.6663376\ttotal: 329ms\tremaining: 5.16s\n",
      "6:\tlearn: 0.5924449\ttotal: 384ms\tremaining: 5.1s\n",
      "7:\tlearn: 0.5333106\ttotal: 436ms\tremaining: 5.02s\n",
      "8:\tlearn: 0.4812230\ttotal: 489ms\tremaining: 4.94s\n",
      "9:\tlearn: 0.4376355\ttotal: 542ms\tremaining: 4.88s\n",
      "10:\tlearn: 0.3963971\ttotal: 594ms\tremaining: 4.8s\n",
      "11:\tlearn: 0.3626403\ttotal: 647ms\tremaining: 4.74s\n",
      "12:\tlearn: 0.3319598\ttotal: 698ms\tremaining: 4.67s\n",
      "13:\tlearn: 0.3054207\ttotal: 751ms\tremaining: 4.61s\n",
      "14:\tlearn: 0.2807111\ttotal: 805ms\tremaining: 4.56s\n",
      "15:\tlearn: 0.2581421\ttotal: 853ms\tremaining: 4.48s\n",
      "16:\tlearn: 0.2391757\ttotal: 906ms\tremaining: 4.42s\n",
      "17:\tlearn: 0.2218704\ttotal: 954ms\tremaining: 4.34s\n",
      "18:\tlearn: 0.2064877\ttotal: 1.01s\tremaining: 4.29s\n",
      "19:\tlearn: 0.1915146\ttotal: 1.05s\tremaining: 4.22s\n",
      "20:\tlearn: 0.1793644\ttotal: 1.1s\tremaining: 4.15s\n",
      "21:\tlearn: 0.1678457\ttotal: 1.16s\tremaining: 4.1s\n",
      "22:\tlearn: 0.1566736\ttotal: 1.2s\tremaining: 4.03s\n",
      "23:\tlearn: 0.1471154\ttotal: 1.25s\tremaining: 3.97s\n",
      "24:\tlearn: 0.1386685\ttotal: 1.3s\tremaining: 3.91s\n",
      "25:\tlearn: 0.1315447\ttotal: 1.35s\tremaining: 3.85s\n",
      "26:\tlearn: 0.1234937\ttotal: 1.41s\tremaining: 3.81s\n",
      "27:\tlearn: 0.1158532\ttotal: 1.46s\tremaining: 3.75s\n",
      "28:\tlearn: 0.1091006\ttotal: 1.51s\tremaining: 3.7s\n",
      "29:\tlearn: 0.1018262\ttotal: 1.56s\tremaining: 3.65s\n",
      "30:\tlearn: 0.0960930\ttotal: 1.61s\tremaining: 3.59s\n",
      "31:\tlearn: 0.0907188\ttotal: 1.66s\tremaining: 3.54s\n",
      "32:\tlearn: 0.0871515\ttotal: 1.71s\tremaining: 3.47s\n",
      "33:\tlearn: 0.0827720\ttotal: 1.76s\tremaining: 3.41s\n",
      "34:\tlearn: 0.0788598\ttotal: 1.81s\tremaining: 3.35s\n",
      "35:\tlearn: 0.0767508\ttotal: 1.85s\tremaining: 3.28s\n",
      "36:\tlearn: 0.0728872\ttotal: 1.9s\tremaining: 3.23s\n",
      "37:\tlearn: 0.0693075\ttotal: 1.95s\tremaining: 3.18s\n",
      "38:\tlearn: 0.0659997\ttotal: 2s\tremaining: 3.12s\n",
      "39:\tlearn: 0.0631960\ttotal: 2.04s\tremaining: 3.06s\n",
      "40:\tlearn: 0.0616546\ttotal: 2.09s\tremaining: 3.01s\n",
      "41:\tlearn: 0.0591252\ttotal: 2.14s\tremaining: 2.95s\n",
      "42:\tlearn: 0.0569423\ttotal: 2.19s\tremaining: 2.9s\n",
      "43:\tlearn: 0.0555695\ttotal: 2.23s\tremaining: 2.84s\n",
      "44:\tlearn: 0.0544288\ttotal: 2.28s\tremaining: 2.78s\n",
      "45:\tlearn: 0.0529599\ttotal: 2.32s\tremaining: 2.73s\n",
      "46:\tlearn: 0.0511723\ttotal: 2.37s\tremaining: 2.68s\n",
      "47:\tlearn: 0.0492577\ttotal: 2.42s\tremaining: 2.63s\n",
      "48:\tlearn: 0.0474565\ttotal: 2.48s\tremaining: 2.58s\n",
      "49:\tlearn: 0.0461775\ttotal: 2.52s\tremaining: 2.52s\n",
      "50:\tlearn: 0.0447324\ttotal: 2.57s\tremaining: 2.47s\n",
      "51:\tlearn: 0.0441578\ttotal: 2.62s\tremaining: 2.41s\n",
      "52:\tlearn: 0.0426936\ttotal: 2.67s\tremaining: 2.37s\n",
      "53:\tlearn: 0.0411880\ttotal: 2.72s\tremaining: 2.32s\n",
      "54:\tlearn: 0.0398686\ttotal: 2.78s\tremaining: 2.27s\n",
      "55:\tlearn: 0.0392087\ttotal: 2.82s\tremaining: 2.22s\n",
      "56:\tlearn: 0.0380360\ttotal: 2.87s\tremaining: 2.17s\n",
      "57:\tlearn: 0.0368416\ttotal: 2.92s\tremaining: 2.12s\n",
      "58:\tlearn: 0.0359994\ttotal: 2.97s\tremaining: 2.06s\n",
      "59:\tlearn: 0.0349010\ttotal: 3.02s\tremaining: 2.02s\n",
      "60:\tlearn: 0.0340280\ttotal: 3.07s\tremaining: 1.96s\n",
      "61:\tlearn: 0.0332447\ttotal: 3.12s\tremaining: 1.91s\n",
      "62:\tlearn: 0.0324979\ttotal: 3.17s\tremaining: 1.86s\n",
      "63:\tlearn: 0.0316858\ttotal: 3.22s\tremaining: 1.81s\n",
      "64:\tlearn: 0.0310127\ttotal: 3.27s\tremaining: 1.76s\n",
      "65:\tlearn: 0.0306307\ttotal: 3.31s\tremaining: 1.71s\n",
      "66:\tlearn: 0.0299716\ttotal: 3.36s\tremaining: 1.66s\n",
      "67:\tlearn: 0.0294135\ttotal: 3.41s\tremaining: 1.6s\n",
      "68:\tlearn: 0.0292012\ttotal: 3.45s\tremaining: 1.55s\n",
      "69:\tlearn: 0.0287113\ttotal: 3.5s\tremaining: 1.5s\n",
      "70:\tlearn: 0.0280744\ttotal: 3.55s\tremaining: 1.45s\n",
      "71:\tlearn: 0.0276357\ttotal: 3.6s\tremaining: 1.4s\n",
      "72:\tlearn: 0.0269677\ttotal: 3.65s\tremaining: 1.35s\n",
      "73:\tlearn: 0.0266717\ttotal: 3.69s\tremaining: 1.3s\n",
      "74:\tlearn: 0.0262552\ttotal: 3.74s\tremaining: 1.25s\n",
      "75:\tlearn: 0.0257181\ttotal: 3.79s\tremaining: 1.2s\n",
      "76:\tlearn: 0.0252525\ttotal: 3.84s\tremaining: 1.15s\n",
      "77:\tlearn: 0.0249085\ttotal: 3.88s\tremaining: 1.09s\n",
      "78:\tlearn: 0.0245081\ttotal: 3.93s\tremaining: 1.04s\n",
      "79:\tlearn: 0.0242788\ttotal: 3.98s\tremaining: 995ms\n",
      "80:\tlearn: 0.0240343\ttotal: 4.02s\tremaining: 944ms\n",
      "81:\tlearn: 0.0237063\ttotal: 4.07s\tremaining: 893ms\n",
      "82:\tlearn: 0.0235421\ttotal: 4.11s\tremaining: 843ms\n",
      "83:\tlearn: 0.0233049\ttotal: 4.16s\tremaining: 792ms\n",
      "84:\tlearn: 0.0230467\ttotal: 4.21s\tremaining: 742ms\n",
      "85:\tlearn: 0.0226861\ttotal: 4.25s\tremaining: 692ms\n",
      "86:\tlearn: 0.0224024\ttotal: 4.3s\tremaining: 643ms\n",
      "87:\tlearn: 0.0220306\ttotal: 4.35s\tremaining: 593ms\n",
      "88:\tlearn: 0.0215951\ttotal: 4.4s\tremaining: 544ms\n",
      "89:\tlearn: 0.0212216\ttotal: 4.45s\tremaining: 495ms\n",
      "90:\tlearn: 0.0208390\ttotal: 4.5s\tremaining: 445ms\n",
      "91:\tlearn: 0.0206469\ttotal: 4.54s\tremaining: 395ms\n",
      "92:\tlearn: 0.0201694\ttotal: 4.6s\tremaining: 346ms\n",
      "93:\tlearn: 0.0200285\ttotal: 4.64s\tremaining: 296ms\n",
      "94:\tlearn: 0.0198322\ttotal: 4.69s\tremaining: 247ms\n",
      "95:\tlearn: 0.0195336\ttotal: 4.74s\tremaining: 198ms\n",
      "96:\tlearn: 0.0193602\ttotal: 4.78s\tremaining: 148ms\n",
      "97:\tlearn: 0.0190927\ttotal: 4.83s\tremaining: 98.6ms\n",
      "98:\tlearn: 0.0187368\ttotal: 4.88s\tremaining: 49.3ms\n",
      "99:\tlearn: 0.0186445\ttotal: 4.92s\tremaining: 0us\n",
      "0:\tlearn: 1.4403383\ttotal: 56ms\tremaining: 5.54s\n",
      "1:\tlearn: 1.1722923\ttotal: 108ms\tremaining: 5.27s\n",
      "2:\tlearn: 0.9934480\ttotal: 160ms\tremaining: 5.19s\n",
      "3:\tlearn: 0.8559084\ttotal: 212ms\tremaining: 5.09s\n",
      "4:\tlearn: 0.7502223\ttotal: 265ms\tremaining: 5.03s\n",
      "5:\tlearn: 0.6638695\ttotal: 317ms\tremaining: 4.97s\n",
      "6:\tlearn: 0.5900209\ttotal: 374ms\tremaining: 4.96s\n",
      "7:\tlearn: 0.5307346\ttotal: 425ms\tremaining: 4.88s\n",
      "8:\tlearn: 0.4785482\ttotal: 477ms\tremaining: 4.82s\n",
      "9:\tlearn: 0.4353805\ttotal: 528ms\tremaining: 4.75s\n",
      "10:\tlearn: 0.3948525\ttotal: 579ms\tremaining: 4.69s\n",
      "11:\tlearn: 0.3597690\ttotal: 630ms\tremaining: 4.62s\n",
      "12:\tlearn: 0.3303887\ttotal: 682ms\tremaining: 4.57s\n",
      "13:\tlearn: 0.3029506\ttotal: 733ms\tremaining: 4.5s\n",
      "14:\tlearn: 0.2786543\ttotal: 786ms\tremaining: 4.45s\n",
      "15:\tlearn: 0.2565864\ttotal: 835ms\tremaining: 4.38s\n",
      "16:\tlearn: 0.2380736\ttotal: 887ms\tremaining: 4.33s\n",
      "17:\tlearn: 0.2203837\ttotal: 937ms\tremaining: 4.27s\n",
      "18:\tlearn: 0.2061949\ttotal: 986ms\tremaining: 4.2s\n",
      "19:\tlearn: 0.1915890\ttotal: 1.03s\tremaining: 4.13s\n",
      "20:\tlearn: 0.1771292\ttotal: 1.08s\tremaining: 4.07s\n",
      "21:\tlearn: 0.1657058\ttotal: 1.14s\tremaining: 4.03s\n",
      "22:\tlearn: 0.1541454\ttotal: 1.18s\tremaining: 3.96s\n",
      "23:\tlearn: 0.1444225\ttotal: 1.23s\tremaining: 3.89s\n",
      "24:\tlearn: 0.1357592\ttotal: 1.28s\tremaining: 3.84s\n",
      "25:\tlearn: 0.1278262\ttotal: 1.33s\tremaining: 3.78s\n",
      "26:\tlearn: 0.1200452\ttotal: 1.38s\tremaining: 3.72s\n",
      "27:\tlearn: 0.1133124\ttotal: 1.43s\tremaining: 3.67s\n",
      "28:\tlearn: 0.1066203\ttotal: 1.48s\tremaining: 3.61s\n",
      "29:\tlearn: 0.1012485\ttotal: 1.53s\tremaining: 3.56s\n",
      "30:\tlearn: 0.0955274\ttotal: 1.57s\tremaining: 3.5s\n",
      "31:\tlearn: 0.0904887\ttotal: 1.62s\tremaining: 3.45s\n",
      "32:\tlearn: 0.0857382\ttotal: 1.67s\tremaining: 3.4s\n",
      "33:\tlearn: 0.0812362\ttotal: 1.72s\tremaining: 3.34s\n",
      "34:\tlearn: 0.0779285\ttotal: 1.77s\tremaining: 3.29s\n",
      "35:\tlearn: 0.0741198\ttotal: 1.82s\tremaining: 3.23s\n",
      "36:\tlearn: 0.0705743\ttotal: 1.89s\tremaining: 3.21s\n",
      "37:\tlearn: 0.0665223\ttotal: 1.94s\tremaining: 3.16s\n",
      "38:\tlearn: 0.0649978\ttotal: 1.98s\tremaining: 3.09s\n",
      "39:\tlearn: 0.0626543\ttotal: 2.02s\tremaining: 3.04s\n",
      "40:\tlearn: 0.0605910\ttotal: 2.07s\tremaining: 2.98s\n",
      "41:\tlearn: 0.0584397\ttotal: 2.12s\tremaining: 2.92s\n",
      "42:\tlearn: 0.0562523\ttotal: 2.16s\tremaining: 2.87s\n",
      "43:\tlearn: 0.0543594\ttotal: 2.21s\tremaining: 2.81s\n",
      "44:\tlearn: 0.0520387\ttotal: 2.26s\tremaining: 2.76s\n",
      "45:\tlearn: 0.0508875\ttotal: 2.3s\tremaining: 2.7s\n",
      "46:\tlearn: 0.0495847\ttotal: 2.35s\tremaining: 2.64s\n",
      "47:\tlearn: 0.0478179\ttotal: 2.39s\tremaining: 2.59s\n",
      "48:\tlearn: 0.0456579\ttotal: 2.45s\tremaining: 2.55s\n",
      "49:\tlearn: 0.0447147\ttotal: 2.49s\tremaining: 2.49s\n",
      "50:\tlearn: 0.0434975\ttotal: 2.53s\tremaining: 2.43s\n",
      "51:\tlearn: 0.0423668\ttotal: 2.58s\tremaining: 2.38s\n",
      "52:\tlearn: 0.0410138\ttotal: 2.63s\tremaining: 2.34s\n",
      "53:\tlearn: 0.0396306\ttotal: 2.68s\tremaining: 2.29s\n",
      "54:\tlearn: 0.0387644\ttotal: 2.73s\tremaining: 2.24s\n",
      "55:\tlearn: 0.0377967\ttotal: 2.78s\tremaining: 2.18s\n",
      "56:\tlearn: 0.0367641\ttotal: 2.83s\tremaining: 2.13s\n",
      "57:\tlearn: 0.0356266\ttotal: 2.88s\tremaining: 2.08s\n",
      "58:\tlearn: 0.0346685\ttotal: 2.92s\tremaining: 2.03s\n",
      "59:\tlearn: 0.0335998\ttotal: 2.97s\tremaining: 1.98s\n",
      "60:\tlearn: 0.0328676\ttotal: 3.02s\tremaining: 1.93s\n",
      "61:\tlearn: 0.0321245\ttotal: 3.06s\tremaining: 1.88s\n",
      "62:\tlearn: 0.0314739\ttotal: 3.11s\tremaining: 1.83s\n",
      "63:\tlearn: 0.0307809\ttotal: 3.16s\tremaining: 1.78s\n",
      "64:\tlearn: 0.0300164\ttotal: 3.2s\tremaining: 1.73s\n",
      "65:\tlearn: 0.0292870\ttotal: 3.25s\tremaining: 1.68s\n",
      "66:\tlearn: 0.0289284\ttotal: 3.3s\tremaining: 1.62s\n",
      "67:\tlearn: 0.0283228\ttotal: 3.34s\tremaining: 1.57s\n",
      "68:\tlearn: 0.0281649\ttotal: 3.38s\tremaining: 1.52s\n",
      "69:\tlearn: 0.0273677\ttotal: 3.44s\tremaining: 1.47s\n",
      "70:\tlearn: 0.0271727\ttotal: 3.48s\tremaining: 1.42s\n",
      "71:\tlearn: 0.0265242\ttotal: 3.53s\tremaining: 1.37s\n",
      "72:\tlearn: 0.0260978\ttotal: 3.58s\tremaining: 1.32s\n",
      "73:\tlearn: 0.0256606\ttotal: 3.63s\tremaining: 1.27s\n",
      "74:\tlearn: 0.0252212\ttotal: 3.68s\tremaining: 1.23s\n",
      "75:\tlearn: 0.0247041\ttotal: 3.73s\tremaining: 1.18s\n",
      "76:\tlearn: 0.0243174\ttotal: 3.77s\tremaining: 1.13s\n",
      "77:\tlearn: 0.0238558\ttotal: 3.82s\tremaining: 1.08s\n",
      "78:\tlearn: 0.0234840\ttotal: 3.87s\tremaining: 1.03s\n",
      "79:\tlearn: 0.0231027\ttotal: 3.92s\tremaining: 980ms\n",
      "80:\tlearn: 0.0228484\ttotal: 3.96s\tremaining: 930ms\n",
      "81:\tlearn: 0.0226086\ttotal: 4.01s\tremaining: 880ms\n",
      "82:\tlearn: 0.0223421\ttotal: 4.06s\tremaining: 831ms\n",
      "83:\tlearn: 0.0219860\ttotal: 4.11s\tremaining: 782ms\n",
      "84:\tlearn: 0.0215699\ttotal: 4.16s\tremaining: 734ms\n",
      "85:\tlearn: 0.0213269\ttotal: 4.2s\tremaining: 684ms\n",
      "86:\tlearn: 0.0210510\ttotal: 4.25s\tremaining: 635ms\n",
      "87:\tlearn: 0.0206868\ttotal: 4.3s\tremaining: 586ms\n",
      "88:\tlearn: 0.0204704\ttotal: 4.34s\tremaining: 537ms\n",
      "89:\tlearn: 0.0203448\ttotal: 4.39s\tremaining: 488ms\n",
      "90:\tlearn: 0.0201056\ttotal: 4.45s\tremaining: 440ms\n",
      "91:\tlearn: 0.0198790\ttotal: 4.49s\tremaining: 391ms\n",
      "92:\tlearn: 0.0193767\ttotal: 4.55s\tremaining: 342ms\n",
      "93:\tlearn: 0.0191106\ttotal: 4.59s\tremaining: 293ms\n",
      "94:\tlearn: 0.0189044\ttotal: 4.64s\tremaining: 244ms\n",
      "95:\tlearn: 0.0187312\ttotal: 4.68s\tremaining: 195ms\n",
      "96:\tlearn: 0.0185768\ttotal: 4.73s\tremaining: 146ms\n",
      "97:\tlearn: 0.0182755\ttotal: 4.78s\tremaining: 97.5ms\n",
      "98:\tlearn: 0.0179892\ttotal: 4.83s\tremaining: 48.7ms\n",
      "99:\tlearn: 0.0178603\ttotal: 4.87s\tremaining: 0us\n",
      "0:\tlearn: 1.4411957\ttotal: 65.9ms\tremaining: 6.52s\n",
      "1:\tlearn: 1.1768802\ttotal: 116ms\tremaining: 5.68s\n",
      "2:\tlearn: 0.9976267\ttotal: 167ms\tremaining: 5.4s\n",
      "3:\tlearn: 0.8606786\ttotal: 218ms\tremaining: 5.23s\n",
      "4:\tlearn: 0.7548451\ttotal: 270ms\tremaining: 5.12s\n",
      "5:\tlearn: 0.6685156\ttotal: 323ms\tremaining: 5.06s\n",
      "6:\tlearn: 0.5943208\ttotal: 376ms\tremaining: 4.99s\n",
      "7:\tlearn: 0.5351225\ttotal: 424ms\tremaining: 4.88s\n",
      "8:\tlearn: 0.4828887\ttotal: 475ms\tremaining: 4.8s\n",
      "9:\tlearn: 0.4406178\ttotal: 526ms\tremaining: 4.74s\n",
      "10:\tlearn: 0.3990637\ttotal: 576ms\tremaining: 4.66s\n",
      "11:\tlearn: 0.3641365\ttotal: 627ms\tremaining: 4.6s\n",
      "12:\tlearn: 0.3334967\ttotal: 677ms\tremaining: 4.53s\n",
      "13:\tlearn: 0.3057974\ttotal: 727ms\tremaining: 4.46s\n",
      "14:\tlearn: 0.2811564\ttotal: 779ms\tremaining: 4.41s\n",
      "15:\tlearn: 0.2590785\ttotal: 828ms\tremaining: 4.35s\n",
      "16:\tlearn: 0.2398590\ttotal: 879ms\tremaining: 4.29s\n",
      "17:\tlearn: 0.2229929\ttotal: 930ms\tremaining: 4.24s\n",
      "18:\tlearn: 0.2085185\ttotal: 978ms\tremaining: 4.17s\n",
      "19:\tlearn: 0.1934984\ttotal: 1.02s\tremaining: 4.1s\n",
      "20:\tlearn: 0.1792547\ttotal: 1.07s\tremaining: 4.03s\n",
      "21:\tlearn: 0.1682604\ttotal: 1.12s\tremaining: 3.98s\n",
      "22:\tlearn: 0.1566529\ttotal: 1.17s\tremaining: 3.91s\n",
      "23:\tlearn: 0.1466001\ttotal: 1.22s\tremaining: 3.86s\n",
      "24:\tlearn: 0.1373774\ttotal: 1.27s\tremaining: 3.8s\n",
      "25:\tlearn: 0.1293728\ttotal: 1.31s\tremaining: 3.74s\n",
      "26:\tlearn: 0.1215917\ttotal: 1.36s\tremaining: 3.69s\n",
      "27:\tlearn: 0.1138635\ttotal: 1.41s\tremaining: 3.63s\n",
      "28:\tlearn: 0.1073625\ttotal: 1.46s\tremaining: 3.57s\n",
      "29:\tlearn: 0.1018318\ttotal: 1.51s\tremaining: 3.52s\n",
      "30:\tlearn: 0.0963139\ttotal: 1.55s\tremaining: 3.46s\n",
      "31:\tlearn: 0.0909762\ttotal: 1.6s\tremaining: 3.41s\n",
      "32:\tlearn: 0.0862156\ttotal: 1.65s\tremaining: 3.36s\n",
      "33:\tlearn: 0.0817177\ttotal: 1.7s\tremaining: 3.31s\n",
      "34:\tlearn: 0.0766728\ttotal: 1.75s\tremaining: 3.25s\n",
      "35:\tlearn: 0.0740402\ttotal: 1.79s\tremaining: 3.19s\n",
      "36:\tlearn: 0.0709613\ttotal: 1.84s\tremaining: 3.14s\n",
      "37:\tlearn: 0.0674936\ttotal: 1.89s\tremaining: 3.09s\n",
      "38:\tlearn: 0.0643937\ttotal: 1.94s\tremaining: 3.04s\n",
      "39:\tlearn: 0.0618372\ttotal: 1.99s\tremaining: 2.98s\n",
      "40:\tlearn: 0.0592928\ttotal: 2.04s\tremaining: 2.93s\n",
      "41:\tlearn: 0.0568488\ttotal: 2.09s\tremaining: 2.88s\n",
      "42:\tlearn: 0.0550541\ttotal: 2.13s\tremaining: 2.82s\n",
      "43:\tlearn: 0.0533619\ttotal: 2.17s\tremaining: 2.77s\n",
      "44:\tlearn: 0.0525104\ttotal: 2.21s\tremaining: 2.71s\n",
      "45:\tlearn: 0.0511271\ttotal: 2.26s\tremaining: 2.65s\n",
      "46:\tlearn: 0.0494190\ttotal: 2.3s\tremaining: 2.6s\n",
      "47:\tlearn: 0.0478042\ttotal: 2.35s\tremaining: 2.54s\n",
      "48:\tlearn: 0.0464014\ttotal: 2.4s\tremaining: 2.49s\n",
      "49:\tlearn: 0.0451108\ttotal: 2.44s\tremaining: 2.44s\n",
      "50:\tlearn: 0.0438495\ttotal: 2.49s\tremaining: 2.39s\n",
      "51:\tlearn: 0.0421616\ttotal: 2.54s\tremaining: 2.34s\n",
      "52:\tlearn: 0.0408993\ttotal: 2.58s\tremaining: 2.29s\n",
      "53:\tlearn: 0.0397579\ttotal: 2.64s\tremaining: 2.25s\n",
      "54:\tlearn: 0.0382803\ttotal: 2.69s\tremaining: 2.2s\n",
      "55:\tlearn: 0.0373836\ttotal: 2.73s\tremaining: 2.15s\n",
      "56:\tlearn: 0.0363781\ttotal: 2.78s\tremaining: 2.1s\n",
      "57:\tlearn: 0.0356203\ttotal: 2.82s\tremaining: 2.04s\n",
      "58:\tlearn: 0.0351764\ttotal: 2.86s\tremaining: 1.99s\n",
      "59:\tlearn: 0.0339745\ttotal: 2.92s\tremaining: 1.94s\n",
      "60:\tlearn: 0.0330776\ttotal: 2.96s\tremaining: 1.89s\n",
      "61:\tlearn: 0.0321750\ttotal: 3.01s\tremaining: 1.84s\n",
      "62:\tlearn: 0.0315311\ttotal: 3.06s\tremaining: 1.8s\n",
      "63:\tlearn: 0.0306140\ttotal: 3.11s\tremaining: 1.75s\n",
      "64:\tlearn: 0.0299332\ttotal: 3.16s\tremaining: 1.7s\n",
      "65:\tlearn: 0.0293804\ttotal: 3.21s\tremaining: 1.65s\n",
      "66:\tlearn: 0.0287964\ttotal: 3.25s\tremaining: 1.6s\n",
      "67:\tlearn: 0.0281190\ttotal: 3.31s\tremaining: 1.55s\n",
      "68:\tlearn: 0.0277472\ttotal: 3.35s\tremaining: 1.51s\n",
      "69:\tlearn: 0.0269974\ttotal: 3.4s\tremaining: 1.46s\n",
      "70:\tlearn: 0.0264799\ttotal: 3.46s\tremaining: 1.41s\n",
      "71:\tlearn: 0.0259635\ttotal: 3.5s\tremaining: 1.36s\n",
      "72:\tlearn: 0.0255362\ttotal: 3.55s\tremaining: 1.31s\n",
      "73:\tlearn: 0.0249850\ttotal: 3.59s\tremaining: 1.26s\n",
      "74:\tlearn: 0.0246785\ttotal: 3.63s\tremaining: 1.21s\n",
      "75:\tlearn: 0.0240838\ttotal: 3.68s\tremaining: 1.16s\n",
      "76:\tlearn: 0.0235657\ttotal: 3.73s\tremaining: 1.11s\n",
      "77:\tlearn: 0.0231512\ttotal: 3.77s\tremaining: 1.06s\n",
      "78:\tlearn: 0.0227400\ttotal: 3.82s\tremaining: 1.01s\n",
      "79:\tlearn: 0.0223780\ttotal: 3.87s\tremaining: 968ms\n",
      "80:\tlearn: 0.0220334\ttotal: 3.92s\tremaining: 919ms\n",
      "81:\tlearn: 0.0218415\ttotal: 3.96s\tremaining: 870ms\n",
      "82:\tlearn: 0.0214702\ttotal: 4.01s\tremaining: 822ms\n",
      "83:\tlearn: 0.0211267\ttotal: 4.06s\tremaining: 774ms\n",
      "84:\tlearn: 0.0208269\ttotal: 4.11s\tremaining: 725ms\n",
      "85:\tlearn: 0.0204816\ttotal: 4.16s\tremaining: 676ms\n",
      "86:\tlearn: 0.0202422\ttotal: 4.2s\tremaining: 628ms\n",
      "87:\tlearn: 0.0199488\ttotal: 4.25s\tremaining: 579ms\n",
      "88:\tlearn: 0.0196650\ttotal: 4.3s\tremaining: 531ms\n",
      "89:\tlearn: 0.0194480\ttotal: 4.34s\tremaining: 483ms\n",
      "90:\tlearn: 0.0192290\ttotal: 4.39s\tremaining: 434ms\n",
      "91:\tlearn: 0.0189097\ttotal: 4.44s\tremaining: 386ms\n",
      "92:\tlearn: 0.0186252\ttotal: 4.49s\tremaining: 338ms\n",
      "93:\tlearn: 0.0183720\ttotal: 4.53s\tremaining: 289ms\n",
      "94:\tlearn: 0.0180979\ttotal: 4.58s\tremaining: 241ms\n",
      "95:\tlearn: 0.0178443\ttotal: 4.63s\tremaining: 193ms\n",
      "96:\tlearn: 0.0176383\ttotal: 4.68s\tremaining: 145ms\n",
      "97:\tlearn: 0.0174313\ttotal: 4.73s\tremaining: 96.5ms\n",
      "98:\tlearn: 0.0171924\ttotal: 4.78s\tremaining: 48.3ms\n",
      "99:\tlearn: 0.0170776\ttotal: 4.82s\tremaining: 0us\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0       1      2      3      4      5        6\n",
      "0  12645.0     0.0    0.0    0.0    2.0    0.0     37.0\n",
      "1      4.0  5259.0    0.0    0.0    3.0    0.0      8.0\n",
      "2      0.0     0.0  417.0    0.0    0.0    0.0      2.0\n",
      "3      0.0     0.0    0.0  451.0    0.0    0.0     23.0\n",
      "4      0.0     0.0    0.0    0.0  448.0    0.0      1.0\n",
      "5      0.0     0.0    0.0    0.0    0.0  468.0      3.0\n",
      "6     42.0    12.0    0.0   15.0    0.0    0.0  37704.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9973585430279438\n",
      "Precision total:  0.9927019422105235\n",
      "Recall total:  0.9900516204005776\n",
      "F1 total:  0.9913617686628735\n",
      "BACC total:  0.9900516204005776\n",
      "MCC total:  0.9948389598635486\n"
     ]
    }
   ],
   "source": [
    "import catboost\n",
    "start = time.time()\n",
    "\n",
    "bag_cat = catboost.CatBoostClassifier(iterations=100, depth=6, learning_rate=0.1, loss_function='MultiClass', custom_metric='Accuracy')\n",
    "\n",
    "base_classifier = bag_cat\n",
    "\n",
    "# Define the BaggingClassifier\n",
    "bagging_classifier = BaggingClassifier(base_classifier, n_estimators=10, random_state=42)\n",
    "\n",
    "# Train the BaggingClassifier\n",
    "bagging_classifier.fit(X_train, y_train)\n",
    "\n",
    "# Make predictions on the test set\n",
    "y_pred = bagging_classifier.predict(X_test)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('--------------------------------------------------------------------------', file = f)\n",
    "\n",
    "name = 'bag_cat'\n",
    "\n",
    "pred_label = y_pred\n",
    "\n",
    "\n",
    "metrics = confusion_metrics(name, pred_label, y_test)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_00\"] = Acc\n",
    "globals()[f\"{name}_pre_00\"] = Precision\n",
    "globals()[f\"{name}_rec_00\"] = Recall\n",
    "globals()[f\"{name}_f1_00\"] = F1\n",
    "globals()[f\"{name}_bacc_00\"] = BACC\n",
    "globals()[f\"{name}_mcc_00\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_00\"] = time_taken\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bagging Combined"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:\tlearn: 1.4340610\ttest: 1.4354507\tbest: 1.4354507 (0)\ttotal: 62.4ms\tremaining: 6.18s\n",
      "10:\tlearn: 0.3911712\ttest: 0.3933428\tbest: 0.3933428 (10)\ttotal: 591ms\tremaining: 4.78s\n",
      "20:\tlearn: 0.1770385\ttest: 0.1785463\tbest: 0.1785463 (20)\ttotal: 1.1s\tremaining: 4.14s\n",
      "30:\tlearn: 0.0971087\ttest: 0.0979704\tbest: 0.0979704 (30)\ttotal: 1.59s\tremaining: 3.54s\n",
      "40:\tlearn: 0.0604976\ttest: 0.0607409\tbest: 0.0607409 (40)\ttotal: 2.09s\tremaining: 3s\n",
      "50:\tlearn: 0.0439405\ttest: 0.0438949\tbest: 0.0438949 (50)\ttotal: 2.56s\tremaining: 2.46s\n",
      "60:\tlearn: 0.0345600\ttest: 0.0342342\tbest: 0.0342342 (60)\ttotal: 3.03s\tremaining: 1.93s\n",
      "70:\tlearn: 0.0279628\ttest: 0.0273973\tbest: 0.0273973 (70)\ttotal: 3.52s\tremaining: 1.44s\n",
      "80:\tlearn: 0.0238007\ttest: 0.0231970\tbest: 0.0231970 (80)\ttotal: 4s\tremaining: 938ms\n",
      "90:\tlearn: 0.0210418\ttest: 0.0203832\tbest: 0.0203832 (90)\ttotal: 4.48s\tremaining: 443ms\n",
      "99:\tlearn: 0.0189192\ttest: 0.0182473\tbest: 0.0182473 (99)\ttotal: 4.91s\tremaining: 0us\n",
      "\n",
      "bestTest = 0.01824734172\n",
      "bestIteration = 99\n",
      "\n",
      "Epoch 1/100\n",
      "840/840 [==============================] - 2s 3ms/step - loss: 1.9973 - accuracy: 0.7070 - val_loss: 0.5123 - val_accuracy: 0.8021\n",
      "Epoch 2/100\n",
      "840/840 [==============================] - 2s 2ms/step - loss: 3.4090 - accuracy: 0.7869 - val_loss: 0.5006 - val_accuracy: 0.8055\n",
      "Epoch 3/100\n",
      "840/840 [==============================] - 2s 2ms/step - loss: 1.8587 - accuracy: 0.7882 - val_loss: 0.4967 - val_accuracy: 0.8048\n",
      "Epoch 4/100\n",
      "840/840 [==============================] - 2s 2ms/step - loss: 1.3104 - accuracy: 0.7880 - val_loss: 0.4906 - val_accuracy: 0.8071\n",
      "Epoch 5/100\n",
      "840/840 [==============================] - 2s 2ms/step - loss: 1.0131 - accuracy: 0.7864 - val_loss: 0.4888 - val_accuracy: 0.8072\n",
      "Epoch 6/100\n",
      "840/840 [==============================] - 2s 2ms/step - loss: 2.2752 - accuracy: 0.7830 - val_loss: 0.4841 - val_accuracy: 0.8106\n",
      "Epoch 7/100\n",
      "840/840 [==============================] - 2s 3ms/step - loss: 2.0027 - accuracy: 0.7843 - val_loss: 0.4967 - val_accuracy: 0.8092\n",
      "Epoch 8/100\n",
      "840/840 [==============================] - 2s 2ms/step - loss: 2.2624 - accuracy: 0.7776 - val_loss: 0.4975 - val_accuracy: 0.8050\n",
      "Epoch 9/100\n",
      "840/840 [==============================] - 2s 2ms/step - loss: 4.7891 - accuracy: 0.7819 - val_loss: 0.5129 - val_accuracy: 0.8012\n",
      "Epoch 10/100\n",
      "840/840 [==============================] - 2s 2ms/step - loss: 1.6151 - accuracy: 0.7732 - val_loss: 0.5070 - val_accuracy: 0.8060\n",
      "Epoch 11/100\n",
      "840/840 [==============================] - 2s 2ms/step - loss: 0.6815 - accuracy: 0.7757 - val_loss: 0.5038 - val_accuracy: 0.8043\n",
      "Epoch 12/100\n",
      "840/840 [==============================] - 2s 2ms/step - loss: 2.2555 - accuracy: 0.7773 - val_loss: 0.5213 - val_accuracy: 0.8047\n",
      "Epoch 13/100\n",
      "840/840 [==============================] - 2s 2ms/step - loss: 0.8569 - accuracy: 0.7780 - val_loss: 0.5173 - val_accuracy: 0.8046\n",
      "Epoch 14/100\n",
      "840/840 [==============================] - 2s 2ms/step - loss: 0.6869 - accuracy: 0.7757 - val_loss: 0.5110 - val_accuracy: 0.8054\n",
      "Epoch 15/100\n",
      "840/840 [==============================] - 2s 2ms/step - loss: 1.5199 - accuracy: 0.7774 - val_loss: 0.5173 - val_accuracy: 0.8048\n",
      "Epoch 16/100\n",
      "840/840 [==============================] - 2s 2ms/step - loss: 0.6896 - accuracy: 0.7738 - val_loss: 0.5041 - val_accuracy: 0.8051\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       model_0  model_1  model_2  model_3  model_4  model_5  model_6  model_7  \\\n",
      "0            6        6        6        6        6        6        6        6   \n",
      "1            6        6        6        6        6        6        6        6   \n",
      "2            6        6        6        6        6        6        6        6   \n",
      "3            6        6        6        6        6        6        6        6   \n",
      "4            6        6        6        6        6        6        6        6   \n",
      "...        ...      ...      ...      ...      ...      ...      ...      ...   \n",
      "57539        0        6        0        0        0        0        0        6   \n",
      "57540        6        6        6        6        6        6        6        6   \n",
      "57541        6        6        6        6        6        6        6        6   \n",
      "57542        6        6        4        4        4        4        6        6   \n",
      "57543        6        6        6        6        6        6        6        6   \n",
      "\n",
      "       model_8  model_9  \n",
      "0            6      6.0  \n",
      "1            6      6.0  \n",
      "2            6      6.0  \n",
      "3            6      6.0  \n",
      "4            6      6.0  \n",
      "...        ...      ...  \n",
      "57539        0      0.0  \n",
      "57540        6      6.0  \n",
      "57541        6      6.0  \n",
      "57542        6      4.0  \n",
      "57543        6      6.0  \n",
      "\n",
      "[57544 rows x 10 columns]\n",
      "       model_0  model_1  model_2  model_3  model_4  model_5  model_6  model_7  \\\n",
      "0            6        6        6        6        6        6        6        6   \n",
      "1            6        6        6        6        6        6        6        6   \n",
      "2            6        6        6        6        6        6        6        6   \n",
      "3            6        6        6        6        6        6        6        6   \n",
      "4            6        6        6        6        6        6        6        6   \n",
      "...        ...      ...      ...      ...      ...      ...      ...      ...   \n",
      "57539        0        6        0        0        0        0        0        6   \n",
      "57540        6        6        6        6        6        6        6        6   \n",
      "57541        6        6        6        6        6        6        6        6   \n",
      "57542        6        6        4        4        4        4        6        6   \n",
      "57543        6        6        6        6        6        6        6        6   \n",
      "\n",
      "       model_8  model_9  ensemble  \n",
      "0            6      6.0         6  \n",
      "1            6      6.0         6  \n",
      "2            6      6.0         6  \n",
      "3            6      6.0         6  \n",
      "4            6      6.0         6  \n",
      "...        ...      ...       ...  \n",
      "57539        0      0.0         0  \n",
      "57540        6      6.0         6  \n",
      "57541        6      6.0         6  \n",
      "57542        6      4.0         4  \n",
      "57543        6      6.0         6  \n",
      "\n",
      "[57544 rows x 11 columns]\n",
      "---------------------------------------------------------------------------------\n",
      "CONFUSION MATRIX\n",
      "---------------------------------------------------------------------------------\n",
      "         0       1      2      3      4      5        6\n",
      "0  12608.0     0.0    0.0    0.0    0.0    0.0     76.0\n",
      "1      4.0  5265.0    0.0    0.0    0.0    0.0      5.0\n",
      "2      0.0     0.0  417.0    0.0    0.0    0.0      2.0\n",
      "3      0.0     0.0    0.0  295.0    0.0    0.0    179.0\n",
      "4      0.0     0.0    0.0    0.0  416.0    0.0     33.0\n",
      "5      0.0     0.0    0.0    0.0    0.0  463.0      8.0\n",
      "6     63.0    37.0    0.0   11.0    0.0    0.0  37662.0\n",
      "---------------------------------------------------------------------------------\n",
      "METRICS\n",
      "---------------------------------------------------------------------------------\n",
      "Accuracy total:  0.9927359933268456\n",
      "Precision total:  0.9919723939546803\n",
      "Recall total:  0.9309244157120963\n",
      "F1 total:  0.9559782294019216\n",
      "BACC total:  0.9309244157120963\n",
      "MCC total:  0.9857773151506969\n"
     ]
    }
   ],
   "source": [
    "### Bagging with many models\n",
    "##### do bootstrapping \n",
    "##### 1. Multiple subsets are created from the original dataset, selecting observations with replacement.\n",
    "\n",
    "start = time.time()\n",
    "\n",
    "num_bootstraps = 10  # Adjust the number of bootstraps as needed\n",
    "\n",
    "original_data_df = X_train.assign(label = y_train)\n",
    "boot_df = []\n",
    "for i in range(0,num_bootstraps): \n",
    "    boot_df.append(original_data_df.sample(frac = 1, replace=True).reset_index(drop=True))\n",
    "\n",
    "# boot_df[5]\n",
    "\n",
    "#### 2.A base model (weak model) is created on each of these subsets.\n",
    "bag_comb_pred = []\n",
    "\n",
    "# SVM\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "clf = SGDClassifier(\n",
    "    loss='hinge',           # hinge loss for linear SVM\n",
    "    penalty='l2',           # L2 regularization to prevent overfitting\n",
    "    alpha=1e-4,             # Learning rate (small value for fine-grained updates)\n",
    "    max_iter=1000,          # Number of passes over the training data\n",
    "    random_state=42,        # Seed for reproducible results\n",
    "    learning_rate='optimal' # Automatically adjusts the learning rate based on the training data\n",
    ")\n",
    "y_train_boot = boot_df[0].pop('label')\n",
    "X_train_boot = boot_df[0]\n",
    "clf.fit(X_train_boot, y_train_boot)\n",
    "preds_svm_00 = clf.predict(X_test)\n",
    "bag_comb_pred.append(preds_svm_00)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "#ADA\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "abc = AdaBoostClassifier(n_estimators=50, learning_rate=1.0)\n",
    "ada = abc.fit(X_train, y_train)\n",
    "y_train_boot = boot_df[1].pop('label')\n",
    "X_train_boot = boot_df[1]\n",
    "preds_ada_00 = ada.predict(X_test)\n",
    "bag_comb_pred.append(preds_ada_00)\n",
    "\n",
    "#Catboost\n",
    "import catboost\n",
    "cat_00 = catboost.CatBoostClassifier(iterations=100, depth=6, learning_rate=0.1, loss_function='MultiClass', custom_metric='Accuracy')\n",
    "y_train_boot = boot_df[2].pop('label')\n",
    "X_train_boot = boot_df[2]\n",
    "cat_00.fit(X_train_boot, y_train_boot, eval_set=(X_test, y_test), verbose=10)\n",
    "preds_cat = cat_00.predict(X_test)\n",
    "preds_cat = np.squeeze(preds_cat)\n",
    "pred_label = preds_cat\n",
    "bag_comb_pred.append(preds_cat)\n",
    "\n",
    "#MLP\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "mlp = MLPClassifier(hidden_layer_sizes=(100,), max_iter=200, random_state=1)\n",
    "y_train_boot = boot_df[3].pop('label')\n",
    "X_train_boot = boot_df[3]\n",
    "if 1 == 1 and 0 == 0:\n",
    "    MLP = mlp.fit(X_train_boot, y_train_boot)\n",
    "    y_pred = MLP.predict_proba(X_test)\n",
    "    preds_mlp_00 = np.argmax(y_pred,axis = 1)\n",
    "\n",
    "bag_comb_pred.append(preds_mlp_00)\n",
    "\n",
    "#LGBM\n",
    "from lightgbm import LGBMClassifier\n",
    "lgbm = LGBMClassifier()\n",
    "y_train_boot = boot_df[4].pop('label')\n",
    "X_train_boot = boot_df[4]\n",
    "\n",
    "if 1 == 1 and 0 == 0:\n",
    "    lgbm.fit(X_train_boot, y_train_boot)\n",
    "    preds_lgbm_00 = lgbm.predict(X_test)\n",
    "    bag_comb_pred.append(preds_lgbm_00)\n",
    "#KNN\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "knn_clf_00=KNeighborsClassifier(n_neighbors = 5)\n",
    "y_train_boot = boot_df[5].pop('label')\n",
    "X_train_boot = boot_df[5]\n",
    "\n",
    "if 1 == 1 and 0 == 0:\n",
    "    knn_clf_00.fit(X_train_boot,y_train_boot)\n",
    "if use_model_knn == 1:\n",
    "    preds_knn =knn_clf_00.predict(X_test)\n",
    "    bag_comb_pred.append(preds_knn)\n",
    "#Random Forest\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "rf = RandomForestClassifier(max_depth = 5,  n_estimators = 10, min_samples_split = 2, n_jobs = -1)\n",
    "y_train_boot = boot_df[6].pop('label')\n",
    "X_train_boot = boot_df[6]\n",
    "\n",
    "if True == True:\n",
    "    model_rf_00 = rf.fit(X_train_boot,y_train_boot)\n",
    "    preds_rf_00 = model_rf_00.predict(X_test)\n",
    "    bag_comb_pred.append(preds_rf_00)\n",
    "#DNN\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense\n",
    "#Model Parameters\n",
    "y_train_boot = boot_df[7].pop('label')\n",
    "X_train_boot = boot_df[7]\n",
    "\n",
    "\n",
    "dropout_rate = 0.2\n",
    "nodes = 3\n",
    "out_layer = 7\n",
    "optimizer='adam'\n",
    "loss='sparse_categorical_crossentropy'\n",
    "epochs=100\n",
    "batch_size=128\n",
    "num_columns = X_train_boot.shape[1]\n",
    "dnn_00 = tf.keras.Sequential()\n",
    "# Input layer\n",
    "dnn_00.add(tf.keras.Input(shape=(num_columns,)))\n",
    "# Dense layers with dropout\n",
    "dnn_00.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_00.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "dnn_00.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_00.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "dnn_00.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_00.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "dnn_00.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_00.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "dnn_00.add(tf.keras.layers.Dense(nodes))\n",
    "dnn_00.add(tf.keras.layers.Dropout(dropout_rate))\n",
    "# Output layer\n",
    "# dnn_00.add(tf.keras.layers.Dense(out_layer))\n",
    "dnn_00.add(tf.keras.layers.Dense(out_layer, activation='softmax'))\n",
    "dnn_00.compile(optimizer=optimizer, loss=loss,metrics=['accuracy'])\n",
    "from keras.callbacks import EarlyStopping\n",
    "# Define EarlyStopping callback\n",
    "early_stopping = EarlyStopping(monitor='val_accuracy', patience=10, restore_best_weights=True)\n",
    "dnn_00.fit(X_train_boot, y_train_boot, epochs=epochs, batch_size=batch_size,validation_split=0.2, callbacks=[early_stopping])\n",
    "pred_dnn = dnn_00.predict(X_test)\n",
    "preds_dnn_00 = np.argmax(pred_dnn,axis = 1)\n",
    "bag_comb_pred.append(preds_dnn_00)\n",
    "#LogReg\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "logreg_00 = LogisticRegression()\n",
    "y_train_boot = boot_df[8].pop('label')\n",
    "X_train_boot = boot_df[8]\n",
    "\n",
    "logreg_00.fit(X_train_boot,y_train_boot)\n",
    "preds_logreg =logreg_00.predict(X_test)\n",
    "bag_comb_pred.append(preds_logreg)\n",
    "import xgboost as xgb\n",
    "y_train_boot = boot_df[9].pop('label')\n",
    "X_train_boot = boot_df[9]\n",
    "\n",
    "# Create a DMatrix for XGBoost\n",
    "dtrain = xgb.DMatrix(X_train_boot, label=y_train_boot)\n",
    "dtest = xgb.DMatrix(X_test, label=y_test)\n",
    "# Set XGBoost parameters\n",
    "params = {\n",
    "    'objective': 'multi:softmax',  # for multi-class classification\n",
    "    'num_class': 7,  # specify the number of classes\n",
    "    'max_depth': 3,\n",
    "    'learning_rate': 0.1,\n",
    "    'eval_metric': 'mlogloss'  # metric for multi-class classification\n",
    "}\n",
    "# Train the XGBoost model\n",
    "num_round = 100\n",
    "xgb_00 = xgb.train(params, dtrain, num_round)\n",
    "preds_xgb_00 = xgb_00.predict(dtest)\n",
    "bag_comb_pred.append(preds_xgb_00)\n",
    "### 3. The models run in parallel and are independent of each other.\n",
    "bag_vot_df = pd.DataFrame()\n",
    "for i in range(0,len(bag_comb_pred)):\n",
    "    bag_vot_df[f'model_{i}'] =  bag_comb_pred[i]\n",
    "print(bag_vot_df)\n",
    "# Voting start\n",
    "from scipy.stats import mode\n",
    "# bag_comb_pred_df = pd.DataFrame(bag_comb_pred)\n",
    "# Extract predictions columns\n",
    "\n",
    "# predictions = df[['dnn', 'rf', 'lgbm', 'ada', 'knn', 'mlp', 'svm','cat','xgb']]\n",
    "    # selected_columns = df.loc[:, ~df.columns.isin(['rf'])]\n",
    "predictions = bag_vot_df \n",
    "\n",
    "# predictions = bag_comb_pred_df.loc[:, ~df.columns.isin(['label'])] #df[column_features]\n",
    "\n",
    "# Use the mode function along axis 1 to get the most common prediction for each row\n",
    "ensemble_predictions, _ = mode(predictions.values, axis=1)\n",
    "\n",
    "# Add the ensemble predictions to the DataFrame\n",
    "bag_vot_df['ensemble'] = ensemble_predictions.astype(int)\n",
    "\n",
    "# Display the DataFrame with ensemble predictions\n",
    "print(bag_vot_df)\n",
    "\n",
    "pred_label = bag_vot_df ['ensemble'].values\n",
    "bag_vot_df.pop('ensemble')\n",
    "\n",
    "\n",
    "name='bag_comb'\n",
    "metrics = confusion_metrics(name, pred_label, y_test)\n",
    "\n",
    "Acc = metrics[0]\n",
    "Precision = metrics[1]\n",
    "Recall = metrics[2]\n",
    "F1 = metrics[3]\n",
    "BACC = metrics[4]\n",
    "MCC = metrics[5]    \n",
    "\n",
    "\n",
    "globals()[f\"{name}_acc_00\"] = Acc\n",
    "globals()[f\"{name}_pre_00\"] = Precision\n",
    "globals()[f\"{name}_rec_00\"] = Recall\n",
    "globals()[f\"{name}_f1_00\"] = F1\n",
    "globals()[f\"{name}_bacc_00\"] = BACC\n",
    "globals()[f\"{name}_mcc_00\"] = MCC\n",
    "end = time.time()\n",
    "time_taken = end - start\n",
    "globals()[f\"{name}_time_00\"] = time_taken\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating new dataset for level 01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "57544 57544\n"
     ]
    }
   ],
   "source": [
    "print(len(preds_dnn_prob), len(y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       0\n",
      "0      6\n",
      "1      6\n",
      "2      6\n",
      "3      6\n",
      "4      6\n",
      "...   ..\n",
      "57539  0\n",
      "57540  6\n",
      "57541  6\n",
      "57542  4\n",
      "57543  6\n",
      "\n",
      "[57544 rows x 1 columns]\n"
     ]
    }
   ],
   "source": [
    "print(y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "       index  0\n",
      "0          0  6\n",
      "1          1  6\n",
      "2          2  6\n",
      "3          3  6\n",
      "4          4  6\n",
      "...      ... ..\n",
      "57539  57539  0\n",
      "57540  57540  6\n",
      "57541  57541  6\n",
      "57542  57542  4\n",
      "57543  57543  6\n",
      "\n",
      "[57544 rows x 2 columns]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0            0\n",
       "1            1\n",
       "2            2\n",
       "3            3\n",
       "4            4\n",
       "         ...  \n",
       "57539    57539\n",
       "57540    57540\n",
       "57541    57541\n",
       "57542    57542\n",
       "57543    57543\n",
       "Name: index, Length: 57544, dtype: int64"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test = pd.DataFrame(y_test)\n",
    "df_from_series = y_test\n",
    "y_test_reset_index = df_from_series.reset_index()\n",
    "# y_test2 = y_test.reset_index(inplace=True)\n",
    "print(y_test_reset_index)\n",
    "y_test_reset_index.pop('index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6"
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test_reset_index.values[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds_dnn_2 = []\n",
    "preds_svm_2 = []\n",
    "preds_rf_2 = []\n",
    "preds_mlp_2 = []\n",
    "preds_ada_2 = []\n",
    "preds_knn_2 = []\n",
    "preds_lgbm_2 = []\n",
    "preds_cat_2 = []\n",
    "preds_xgb_2 = []\n",
    "\n",
    "preds_lr_2 = []\n",
    "preds_dt_2 = []\n",
    "\n",
    "for i in range(0,len(preds_dnn_prob)):  \n",
    "    # print(i)\n",
    "    # print(preds_dnn_prob[i][y_test_reset_index.values[i][0]])\n",
    "    preds_dnn_2.append(preds_dnn_prob[i][y_test_reset_index.values[i][0]])\n",
    "    preds_svm_2.append(preds_svm_prob[i][y_test_reset_index.values[i][0]])\n",
    "    preds_rf_2.append(preds_rf_prob[i][y_test_reset_index.values[i][0]])\n",
    "    preds_mlp_2.append(preds_mlp_prob[i][y_test_reset_index.values[i][0]])\n",
    "    preds_ada_2.append(preds_ada_prob[i][y_test_reset_index.values[i][0]])\n",
    "    preds_knn_2.append(preds_knn_prob[i][y_test_reset_index.values[i][0]])\n",
    "    preds_lgbm_2.append(preds_lgbm_prob[i][y_test_reset_index.values[i][0]])\n",
    "    preds_cat_2.append(preds_cat_prob[i][y_test_reset_index.values[i][0]])\n",
    "    preds_xgb_2.append(preds_xgb_prob[i][y_test_reset_index.values[i][0]])\n",
    "    preds_lr_2.append(preds_lr_prob[i][y_test_reset_index.values[i][0]])\n",
    "    preds_dt_2.append(preds_dt_prob[i][y_test_reset_index.values[i][0]])\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[6. 6. 6. ... 6. 6. 6.]\n",
      " [6. 6. 6. ... 6. 6. 6.]\n",
      " [6. 6. 6. ... 6. 6. 6.]\n",
      " ...\n",
      " [6. 6. 6. ... 6. 6. 6.]\n",
      " [6. 6. 4. ... 6. 4. 4.]\n",
      " [6. 6. 6. ... 6. 6. 6.]]\n",
      "[[0.98974633 0.96076917 0.99999986 ... 0.99814604 1.         6.        ]\n",
      " [0.38577598 0.81286454 0.99989118 ... 0.56933003 1.         6.        ]\n",
      " [0.98648167 0.94671146 0.99999948 ... 0.99882637 1.         6.        ]\n",
      " ...\n",
      " [0.98940629 0.94671146 0.99999955 ... 0.99725878 1.         6.        ]\n",
      " [0.01713806 0.15288328 0.99999989 ... 0.42183594 1.         4.        ]\n",
      " [0.66902894 0.94087182 0.99999935 ... 0.96147705 1.         6.        ]]\n"
     ]
    }
   ],
   "source": [
    "with open(output_file_name, \"a\") as f: print('------------------------------------------------------------------', file = f)\n",
    "with open(output_file_name, \"a\") as f: print('------------------------------------------------------------------', file = f)\n",
    "with open(output_file_name, \"a\") as f: print('------------------------------------------------------------------', file = f)\n",
    "\n",
    "with open(output_file_name, \"a\") as f: print('------------START of STRONGER LEARNER - STACK 01 -----------------', file = f)\n",
    "\n",
    "\n",
    "# Stack the vectors horizontally to create a matrix\n",
    "column_features = ['dnn','rf','lgbm','ada','knn','mlp','svm','cat','xgb','lr','dt','label']\n",
    "training_matrix2 = np.column_stack((\n",
    "                          preds_dnn_2,\n",
    "                          preds_rf_2,\n",
    "                          preds_lgbm_2,\n",
    "                          preds_ada_2,\n",
    "                          preds_knn_2, \n",
    "                          preds_mlp_2,\n",
    "                          preds_svm_2,\n",
    "                          preds_cat_2,\n",
    "                          preds_xgb_2,\n",
    "                          preds_lr_2,\n",
    "                          preds_dt_2,\n",
    "                          y_test\n",
    "                          ))\n",
    "\n",
    "training_matrix = np.column_stack((\n",
    "                          preds_dnn,\n",
    "                          preds_rf,\n",
    "                          preds_lgbm,\n",
    "                          preds_ada,\n",
    "                          preds_knn, \n",
    "                          preds_mlp,\n",
    "                          preds_svm,\n",
    "                          preds_cat,\n",
    "                          preds_xgb,\n",
    "                          preds_lr,\n",
    "                          preds_dt,\n",
    "                        #   preds\n",
    "                          y_test\n",
    "                          ))\n",
    "# Print the resulting matrix\n",
    "print(training_matrix)\n",
    "print(training_matrix2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_level_00_0 = pd.DataFrame(training_matrix, columns=column_features)\n",
    "df_level_00_1 = pd.DataFrame(training_matrix2, columns=column_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Assuming df is your DataFrame\n",
    "if feature_selection_bit == 1:\n",
    "\n",
    "    df_level_00_1.to_csv('base_models_prob_feature_selection.csv', index=False)\n",
    "    df_level_00_0.to_csv('base_models_class_feature_selection.csv', index=False)\n",
    "    \n",
    "if feature_selection_bit == 0:\n",
    "\n",
    "    df_level_00_1.to_csv('base_models_prob_all_features.csv', index=False)\n",
    "    df_level_00_0.to_csv('base_models_class_all_features.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dnn</th>\n",
       "      <th>rf</th>\n",
       "      <th>lgbm</th>\n",
       "      <th>ada</th>\n",
       "      <th>knn</th>\n",
       "      <th>mlp</th>\n",
       "      <th>svm</th>\n",
       "      <th>cat</th>\n",
       "      <th>xgb</th>\n",
       "      <th>lr</th>\n",
       "      <th>dt</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.989746</td>\n",
       "      <td>0.960769</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.236073</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.131851</td>\n",
       "      <td>0.999046</td>\n",
       "      <td>0.997208</td>\n",
       "      <td>0.998146</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.385776</td>\n",
       "      <td>0.812865</td>\n",
       "      <td>0.999891</td>\n",
       "      <td>0.236073</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.903242</td>\n",
       "      <td>0.169333</td>\n",
       "      <td>0.982327</td>\n",
       "      <td>0.959897</td>\n",
       "      <td>0.569330</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.986482</td>\n",
       "      <td>0.946711</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.236073</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.132099</td>\n",
       "      <td>0.998706</td>\n",
       "      <td>0.985395</td>\n",
       "      <td>0.998826</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.873801</td>\n",
       "      <td>0.825618</td>\n",
       "      <td>0.999998</td>\n",
       "      <td>0.236073</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.999145</td>\n",
       "      <td>0.143763</td>\n",
       "      <td>0.998424</td>\n",
       "      <td>0.977852</td>\n",
       "      <td>0.722487</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.958421</td>\n",
       "      <td>0.946711</td>\n",
       "      <td>0.999998</td>\n",
       "      <td>0.236073</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.999988</td>\n",
       "      <td>0.263246</td>\n",
       "      <td>0.998465</td>\n",
       "      <td>0.974411</td>\n",
       "      <td>0.975440</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57539</th>\n",
       "      <td>0.695255</td>\n",
       "      <td>0.633869</td>\n",
       "      <td>0.999990</td>\n",
       "      <td>0.227478</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.775769</td>\n",
       "      <td>0.056696</td>\n",
       "      <td>0.971524</td>\n",
       "      <td>0.993232</td>\n",
       "      <td>0.881618</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57540</th>\n",
       "      <td>0.934564</td>\n",
       "      <td>0.946711</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.236073</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.999998</td>\n",
       "      <td>0.447327</td>\n",
       "      <td>0.997240</td>\n",
       "      <td>0.987315</td>\n",
       "      <td>0.976428</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57541</th>\n",
       "      <td>0.989406</td>\n",
       "      <td>0.946711</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.236073</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.131642</td>\n",
       "      <td>0.998634</td>\n",
       "      <td>0.985395</td>\n",
       "      <td>0.997259</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57542</th>\n",
       "      <td>0.017138</td>\n",
       "      <td>0.152883</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.216673</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.820753</td>\n",
       "      <td>0.035202</td>\n",
       "      <td>0.963943</td>\n",
       "      <td>0.996699</td>\n",
       "      <td>0.421836</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57543</th>\n",
       "      <td>0.669029</td>\n",
       "      <td>0.940872</td>\n",
       "      <td>0.999999</td>\n",
       "      <td>0.306757</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.444604</td>\n",
       "      <td>0.998483</td>\n",
       "      <td>0.979464</td>\n",
       "      <td>0.961477</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>57544 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            dnn        rf      lgbm       ada  knn       mlp       svm  \\\n",
       "0      0.989746  0.960769  1.000000  0.236073  1.0  0.999999  0.131851   \n",
       "1      0.385776  0.812865  0.999891  0.236073  1.0  0.903242  0.169333   \n",
       "2      0.986482  0.946711  0.999999  0.236073  1.0  1.000000  0.132099   \n",
       "3      0.873801  0.825618  0.999998  0.236073  1.0  0.999145  0.143763   \n",
       "4      0.958421  0.946711  0.999998  0.236073  1.0  0.999988  0.263246   \n",
       "...         ...       ...       ...       ...  ...       ...       ...   \n",
       "57539  0.695255  0.633869  0.999990  0.227478  1.0  0.775769  0.056696   \n",
       "57540  0.934564  0.946711  1.000000  0.236073  1.0  0.999998  0.447327   \n",
       "57541  0.989406  0.946711  1.000000  0.236073  1.0  1.000000  0.131642   \n",
       "57542  0.017138  0.152883  1.000000  0.216673  1.0  0.820753  0.035202   \n",
       "57543  0.669029  0.940872  0.999999  0.306757  1.0  1.000000  0.444604   \n",
       "\n",
       "            cat       xgb        lr   dt  label  \n",
       "0      0.999046  0.997208  0.998146  1.0    6.0  \n",
       "1      0.982327  0.959897  0.569330  1.0    6.0  \n",
       "2      0.998706  0.985395  0.998826  1.0    6.0  \n",
       "3      0.998424  0.977852  0.722487  1.0    6.0  \n",
       "4      0.998465  0.974411  0.975440  1.0    6.0  \n",
       "...         ...       ...       ...  ...    ...  \n",
       "57539  0.971524  0.993232  0.881618  1.0    0.0  \n",
       "57540  0.997240  0.987315  0.976428  1.0    6.0  \n",
       "57541  0.998634  0.985395  0.997259  1.0    6.0  \n",
       "57542  0.963943  0.996699  0.421836  1.0    4.0  \n",
       "57543  0.998483  0.979464  0.961477  1.0    6.0  \n",
       "\n",
       "[57544 rows x 12 columns]"
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_level_00_1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dnn</th>\n",
       "      <th>rf</th>\n",
       "      <th>lgbm</th>\n",
       "      <th>ada</th>\n",
       "      <th>knn</th>\n",
       "      <th>mlp</th>\n",
       "      <th>svm</th>\n",
       "      <th>cat</th>\n",
       "      <th>xgb</th>\n",
       "      <th>lr</th>\n",
       "      <th>dt</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57539</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57540</th>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57541</th>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57542</th>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57543</th>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>57544 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       dnn   rf  lgbm  ada  knn  mlp  svm  cat  xgb   lr   dt  label\n",
       "0      6.0  6.0   6.0  6.0  6.0  6.0  6.0  6.0  6.0  6.0  6.0    6.0\n",
       "1      6.0  6.0   6.0  6.0  6.0  6.0  6.0  6.0  6.0  6.0  6.0    6.0\n",
       "2      6.0  6.0   6.0  6.0  6.0  6.0  6.0  6.0  6.0  6.0  6.0    6.0\n",
       "3      6.0  6.0   6.0  6.0  6.0  6.0  6.0  6.0  6.0  6.0  6.0    6.0\n",
       "4      6.0  6.0   6.0  6.0  6.0  6.0  6.0  6.0  6.0  6.0  6.0    6.0\n",
       "...    ...  ...   ...  ...  ...  ...  ...  ...  ...  ...  ...    ...\n",
       "57539  0.0  0.0   0.0  6.0  0.0  0.0  0.0  0.0  0.0  0.0  0.0    0.0\n",
       "57540  6.0  6.0   6.0  6.0  6.0  6.0  6.0  6.0  6.0  6.0  6.0    6.0\n",
       "57541  6.0  6.0   6.0  6.0  6.0  6.0  6.0  6.0  6.0  6.0  6.0    6.0\n",
       "57542  6.0  6.0   4.0  6.0  4.0  4.0  6.0  4.0  4.0  6.0  4.0    4.0\n",
       "57543  6.0  6.0   6.0  6.0  6.0  6.0  6.0  6.0  6.0  6.0  6.0    6.0\n",
       "\n",
       "[57544 rows x 12 columns]"
      ]
     },
     "execution_count": 169,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_level_00_0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# y_01 = df_level_01.pop('label')\n",
    "# X_01 = df_level_01\n",
    "# df_level_01 = df_level_01.assign(label = y_01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X_01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# y_01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_level_01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# split = 0.7\n",
    "\n",
    "# X_train_01,X_test_01, y_train_01, y_test_01 = sklearn.model_selection.train_test_split(X_01, y_01, train_size=split)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from tabulate import tabulate\n",
    "\n",
    "# # Assuming data is a 110x4 list, where each row is a sublist\n",
    "# # data =  [[\"Row {} Col {}\".format(i + 1, j + 1) for j in range(4)] for i in range(110)]\n",
    "# data = [[\"\" for _ in range(3)] for _ in range(12)]\n",
    "\n",
    "# # Manually insert data at specific row and column\n",
    "# # data[0][0] = \"ADA\"\n",
    "# # data[1][0] = \"DNN\"\n",
    "# # data[2][0] = \"SVM\"\n",
    "# # data[3][0] = \"ADA\"\n",
    "# # data[4][0] = \"DNN\"\n",
    "# # data[2][0] = \"SVM\"\n",
    "\n",
    "\n",
    "# names_models = ['ADA',\n",
    "#                 'SVM',\n",
    "#                 'DNN',\n",
    "#                 'MLP',\n",
    "#                 'KNN',\n",
    "#                 'CAT',\n",
    "#                 'XGB',\n",
    "#                 'LGBM',\n",
    "#                 'RF',\n",
    "#                 'LR',\n",
    "#                 'VOTING'\n",
    "#                 ]\n",
    "# level_00_f1 = [ada_f1_00,\n",
    "#                 svm_f1_00,\n",
    "#                 dnn_f1_00,\n",
    "#                 mlp_f1_00,\n",
    "#                 knn_f1_00,\n",
    "#                 cat_f1_00,\n",
    "#                 xgb_f1_00,\n",
    "#                 lgbm_f1_00,\n",
    "#                 rf_f1_00,\n",
    "#                 lr_f1_00,\n",
    "#                 voting_f1_00]  \n",
    "\n",
    "                 \n",
    "\n",
    "# for i in range(0,len(names_models)):\n",
    "#     data[i][0] =  names_models[i]\n",
    "#     data[i][1] = level_00_f1[i]\n",
    "#     data[i][2] = level_01_f1[i]\n",
    "\n",
    "\n",
    " \n",
    "# # data[0][1] = ada_acc_00\n",
    "# # data\n",
    "\n",
    "# # Define column headers\n",
    "# headers = [\"F1\", \"Level 00\", \"Level 01\"]\n",
    "\n",
    "# # Print the table\n",
    "# table = tabulate(data, headers=headers, tablefmt=\"grid\")\n",
    "# print(table)\n",
    "# with open(output_file_name, \"a\") as f: print(table, file = f)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# lr_acc_00 = 0 \n",
    "# voting_acc_00 = 0\n",
    "\n",
    "# lr_pre_00 = 0 \n",
    "# voting_pre_00 = 0\n",
    "\n",
    "# lr_rec_00 = 0 \n",
    "# voting_rec_00 = 0\n",
    "\n",
    "# lr_f1_00 = 0 \n",
    "# voting_f1_00 = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| Models   | ACC-00             | PRE-00              | REC-00              | F1-00               |\n",
      "+==========+====================+=====================+=====================+=====================+\n",
      "| ADA      | 0.7975983595161963 | 0.4698254310937875  | 0.3891830420640798  | 0.3615691553814126  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| SVM      | 0.915525510913388  | 0.38291696391955704 | 0.40556784285449793 | 0.39291461481893897 |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| DNN      | 0.8537293201723898 | 0.25312331853645414 | 0.27576935722688567 | 0.263509127637199   |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| MLP      | 0.9804844988182956 | 0.9375934085480729  | 0.935821370794074   | 0.9322961544092099  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| KNN      | 0.9879917975809815 | 0.9656985494985014  | 0.984588633976266   | 0.9749311924824325  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| CAT      | 0.9974106770471292 | 0.9933494718613088  | 0.9882457066934494  | 0.9907825495819125  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| XGB      | 0.997063116919227  | 0.9918199705337617  | 0.9861906693053245  | 0.9889678837786372  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| LGBM     | 0.9992179897122202 | 0.9965412834382235  | 0.9969784028957874  | 0.9967590205828405  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| RF       | 0.9561726678715418 | 0.7032789805275862  | 0.4790844424412081  | 0.505020308018976   |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| LR       | 0.9286632837480884 | 0.5677322709428243  | 0.5304015376210376  | 0.5255594865089178  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| DT       | 0.998192687334909  | 0.9895550845384244  | 0.9933833719898196  | 0.9933833719898196  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| Bag_svm  | 0.9147782566383984 | 0.3822950680895981  | 0.4055818723346916  | 0.3925233519825036  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| Bag_knn  | 0.9882524676769081 | 0.9656675000439711  | 0.9849632912942331  | 0.9750829220826659  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| Bag_DT   | 0.998870429584318  | 0.9934708788688411  | 0.9959563898301874  | 0.9947010875602247  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| Bag_LR   | 0.9283157236201863 | 0.5691816514624338  | 0.5306663871386478  | 0.5255709115360113  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| Bag_mlp  | 0.9808146809398026 | 0.9388990986768562  | 0.935676187571112   | 0.9326349460164448  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| Bag_rf   | 0.9585534547476714 | 0.5605979361544021  | 0.5168334296010132  | 0.5342892371946626  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| Bag_ada  | 0.7983977478103712 | 0.47129835635444745 | 0.4031351197104249  | 0.38263462174576474 |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| Bag_lgbm | 0.9993048797441958 | 0.9971987795024077  | 0.9976158266900941  | 0.9974054849350702  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| Bag_cat  | 0.9973585430279438 | 0.9927019422105235  | 0.9900516204005776  | 0.9913617686628735  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| Bag_comb | 0.9927359933268456 | 0.9919723939546803  | 0.9309244157120963  | 0.9559782294019216  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "|          |                    |                     |                     |                     |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "|          |                    |                     |                     |                     |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "|          |                    |                     |                     |                     |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n"
     ]
    }
   ],
   "source": [
    "from tabulate import tabulate\n",
    "\n",
    "# Assuming data is a 110x4 list, where each row is a sublist\n",
    "# data =  [[\"Row {} Col {}\".format(i + 1, j + 1) for j in range(4)] for i in range(110)]\n",
    "data = [[\"\" for _ in range(5)] for _ in range(24)]\n",
    "\n",
    "# Manually insert data at specific row and column\n",
    "# data[0][0] = \"ADA\"\n",
    "# data[1][0] = \"DNN\"\n",
    "# data[2][0] = \"SVM\"\n",
    "# data[3][0] = \"ADA\"\n",
    "# data[4][0] = \"DNN\"\n",
    "# data[2][0] = \"SVM\"\n",
    "\n",
    "\n",
    "# names_models = ['ADA',\n",
    "#                 'SVM',\n",
    "#                 'DNN',\n",
    "#                 'MLP',\n",
    "#                 'KNN',\n",
    "#                 'CAT',\n",
    "#                 'XGB',\n",
    "#                 'LGBM',\n",
    "#                 'RF',\n",
    "#                 'LR',\n",
    "#                 'VOTING',\n",
    "#                 '   '\n",
    "#                 ]\n",
    "\n",
    "# names_models = ['ADA',\n",
    "#                 'SVM',\n",
    "#                 'DNN',\n",
    "#                 'MLP',\n",
    "#                 'KNN',\n",
    "#                 'CAT',\n",
    "#                 'XGB',\n",
    "#                 'LGBM',\n",
    "#                 'RF',\n",
    "#                 'LR',\n",
    "#                 'DT',\n",
    "#                 # 'VOTING',\n",
    "#                 'Bag_svm',\n",
    "#                 'Bag_knn',\n",
    "#                 'Bag_DT',\n",
    "#                 'Bag_LR',\n",
    "#                 'Bag_mlp',\n",
    "#                 # 'avg',\n",
    "#                 # 'weighed_avg'\n",
    "#                 ]\n",
    "\n",
    "names_models = ['ADA',\n",
    "                'SVM',\n",
    "                'DNN',\n",
    "                'MLP',\n",
    "                'KNN',\n",
    "                'CAT',\n",
    "                'XGB',\n",
    "                'LGBM',\n",
    "                'RF',\n",
    "                'LR',\n",
    "                'DT',\n",
    "                # 'VOTING',\n",
    "                'Bag_svm',\n",
    "                'Bag_knn',\n",
    "                'Bag_DT',\n",
    "                'Bag_LR',\n",
    "                'Bag_mlp',\n",
    "\n",
    "                'Bag_rf',\n",
    "                'Bag_ada',\n",
    "                'Bag_lgbm',\n",
    "                # 'Bag_xgb',\n",
    "                'Bag_cat',\n",
    "                'Bag_comb',\n",
    "\n",
    "                # 'avg',\n",
    "                # 'weighed_avg'\n",
    "                ]\n",
    "\n",
    "\n",
    "level_00_acc = [ada_acc_00,\n",
    "                svm_acc_00,\n",
    "                dnn_acc_00,\n",
    "                mlp_acc_00,\n",
    "                knn_acc_00,\n",
    "                cat_acc_00,\n",
    "                xgb_acc_00,\n",
    "                lgbm_acc_00,\n",
    "                rf_acc_00,\n",
    "                lr_acc_00,\n",
    "                dt_acc_00,\n",
    "                # voting_acc_00,\n",
    "                bag_svm_acc_00,\n",
    "                bag_knn_acc_00,\n",
    "                bag_dt_acc_00,\n",
    "                bag_lr_acc_00,\n",
    "                bag_mlp_acc_00,\n",
    "               \n",
    "                bag_rf_acc_00,\n",
    "                bag_ada_acc_00,\n",
    "                bag_lgbm_acc_00,\n",
    "\n",
    "                bag_cat_acc_00,\n",
    "                bag_comb_acc_00,\n",
    "               \n",
    "               \n",
    "                \n",
    "                # avg_acc_00,\n",
    "                # weighed_avg_acc_00\n",
    "                ]  \n",
    "\n",
    "                # ]  \n",
    "\n",
    "level_00_pre = [ada_pre_00,\n",
    "                svm_pre_00,\n",
    "                dnn_pre_00,\n",
    "                mlp_pre_00,\n",
    "                knn_pre_00,\n",
    "                cat_pre_00,\n",
    "                xgb_pre_00,\n",
    "                lgbm_pre_00,\n",
    "                rf_pre_00,\n",
    "                lr_pre_00,\n",
    "                dt_pre_00,\n",
    "                # voting_pre_00,\n",
    "                bag_svm_pre_00,\n",
    "                bag_knn_pre_00,\n",
    "                bag_dt_pre_00,\n",
    "                bag_lr_pre_00,\n",
    "                bag_mlp_pre_00,\n",
    "\n",
    "                bag_rf_pre_00,\n",
    "                bag_ada_pre_00,\n",
    "                bag_lgbm_pre_00,\n",
    "\n",
    "                bag_cat_pre_00,\n",
    "                bag_comb_pre_00,\n",
    "               \n",
    "                # avg_pre_00,\n",
    "                # weighed_avg_pre_00\n",
    "                ]  \n",
    "\n",
    "level_00_rec = [ada_rec_00,\n",
    "                svm_rec_00,\n",
    "                dnn_rec_00,\n",
    "                mlp_rec_00,\n",
    "                knn_rec_00,\n",
    "                cat_rec_00,\n",
    "                xgb_rec_00,\n",
    "                lgbm_rec_00,\n",
    "                rf_rec_00,\n",
    "                lr_rec_00,\n",
    "                dt_rec_00,\n",
    "                # voting_rec_00,\n",
    "                bag_svm_rec_00,\n",
    "                bag_knn_rec_00,\n",
    "                bag_dt_rec_00,\n",
    "                bag_lr_rec_00,\n",
    "                bag_mlp_rec_00,\n",
    "\n",
    "                bag_rf_rec_00,\n",
    "                bag_ada_rec_00,\n",
    "                bag_lgbm_rec_00,\n",
    "\n",
    "                bag_cat_rec_00,\n",
    "                bag_comb_rec_00,\n",
    "               \n",
    "                # avg_rec_00,\n",
    "                # weighed_avg_rec_00\n",
    "                ]  \n",
    "\n",
    "level_00_f1 = [ada_f1_00,\n",
    "                svm_f1_00,\n",
    "                dnn_f1_00,\n",
    "                mlp_f1_00,\n",
    "                knn_f1_00,\n",
    "                cat_f1_00,\n",
    "                xgb_f1_00,\n",
    "                lgbm_f1_00,\n",
    "                rf_f1_00,\n",
    "                lr_f1_00,\n",
    "                dt_rec_00,\n",
    "                # voting_f1_00,\n",
    "                bag_svm_f1_00,\n",
    "                bag_knn_f1_00,\n",
    "                bag_dt_f1_00,\n",
    "                bag_lr_f1_00,\n",
    "                bag_mlp_f1_00,\n",
    "\n",
    "                bag_rf_f1_00,\n",
    "                bag_ada_f1_00,\n",
    "                bag_lgbm_f1_00,\n",
    "\n",
    "                bag_cat_f1_00,\n",
    "                bag_comb_f1_00,\n",
    "               \n",
    "                # avg_f1_00,\n",
    "                # weighed_avg_f1_00\n",
    "                ]                   \n",
    "\n",
    "for i in range(0,len(names_models)):\n",
    "    data[i][0] =  names_models[i]\n",
    "\n",
    "    data[i][1] = level_00_acc[i]\n",
    "    # data[i][2] = level_01_acc[i]\n",
    "\n",
    "    data[i][2] = level_00_pre[i] \n",
    "    # data[i][4] = level_01_pre[i]\n",
    "\n",
    "    data[i][3] = level_00_rec[i] \n",
    "    # data[i][6] = level_01_rec[i]\n",
    "\n",
    "    data[i][4] = level_00_f1[i]\n",
    "    # data[i][8] = level_01_f1[i]\n",
    "\n",
    "\n",
    "\n",
    "\n",
    " \n",
    "# data[0][1] = ada_acc_00\n",
    "# data\n",
    "\n",
    "# Define column headers\n",
    "# headers = [\"Models\", \"ACC-00\", \" ACC-01\",\"PRE-00\", \" PRE-01\",\"REC-00\", \" REC-01\",\"F1-00\", \" F1-01\",]\n",
    "headers = [\"Models\", \"ACC-00\",\"PRE-00\",\"REC-00\",\"F1-00\"]\n",
    "\n",
    "\n",
    "# Print the table\n",
    "table = tabulate(data, headers=headers, tablefmt=\"grid\")\n",
    "print(table)\n",
    "# with open(output_file_name, \"a\") as f: print(table, file = f)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| Models   | ACC-00             | PRE-00              | REC-00              | F1-00               |\n",
      "+==========+====================+=====================+=====================+=====================+\n",
      "| Bag_lgbm | 0.9993048797441958 | 0.9971987795024077  | 0.9976158266900941  | 0.9974054849350702  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| LGBM     | 0.9992179897122202 | 0.9965412834382235  | 0.9969784028957874  | 0.9967590205828405  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| Bag_DT   | 0.998870429584318  | 0.9934708788688411  | 0.9959563898301874  | 0.9947010875602247  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| DT       | 0.998192687334909  | 0.9895550845384244  | 0.9933833719898196  | 0.9933833719898196  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| Bag_cat  | 0.9973585430279438 | 0.9927019422105235  | 0.9900516204005776  | 0.9913617686628735  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| CAT      | 0.9974106770471292 | 0.9933494718613088  | 0.9882457066934494  | 0.9907825495819125  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| XGB      | 0.997063116919227  | 0.9918199705337617  | 0.9861906693053245  | 0.9889678837786372  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| Bag_knn  | 0.9882524676769081 | 0.9656675000439711  | 0.9849632912942331  | 0.9750829220826659  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| KNN      | 0.9879917975809815 | 0.9656985494985014  | 0.984588633976266   | 0.9749311924824325  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| Bag_comb | 0.9927359933268456 | 0.9919723939546803  | 0.9309244157120963  | 0.9559782294019216  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| Bag_mlp  | 0.9808146809398026 | 0.9388990986768562  | 0.935676187571112   | 0.9326349460164448  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| MLP      | 0.9804844988182956 | 0.9375934085480729  | 0.935821370794074   | 0.9322961544092099  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| Bag_rf   | 0.9585534547476714 | 0.5605979361544021  | 0.5168334296010132  | 0.5342892371946626  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| Bag_LR   | 0.9283157236201863 | 0.5691816514624338  | 0.5306663871386478  | 0.5255709115360113  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| LR       | 0.9286632837480884 | 0.5677322709428243  | 0.5304015376210376  | 0.5255594865089178  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| RF       | 0.9561726678715418 | 0.7032789805275862  | 0.4790844424412081  | 0.505020308018976   |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| SVM      | 0.915525510913388  | 0.38291696391955704 | 0.40556784285449793 | 0.39291461481893897 |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| Bag_svm  | 0.9147782566383984 | 0.3822950680895981  | 0.4055818723346916  | 0.3925233519825036  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| Bag_ada  | 0.7983977478103712 | 0.47129835635444745 | 0.4031351197104249  | 0.38263462174576474 |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| ADA      | 0.7975983595161963 | 0.4698254310937875  | 0.3891830420640798  | 0.3615691553814126  |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "| DNN      | 0.8537293201723898 | 0.25312331853645414 | 0.27576935722688567 | 0.263509127637199   |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "|          |                    |                     |                     |                     |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "|          |                    |                     |                     |                     |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n",
      "|          |                    |                     |                     |                     |\n",
      "+----------+--------------------+---------------------+---------------------+---------------------+\n"
     ]
    }
   ],
   "source": [
    "# Combine data into a list of tuples for sorting\n",
    "model_data = list(zip(names_models, level_00_acc, level_00_pre, level_00_rec, level_00_f1))\n",
    "\n",
    "# Sort by F1-00 score in descending order\n",
    "model_data_sorted = sorted(model_data, key=lambda x: x[4], reverse=True)\n",
    "\n",
    "# Separate the sorted data back into individual lists\n",
    "sorted_names_models, sorted_level_00_acc, sorted_level_00_pre, sorted_level_00_rec, sorted_level_00_f1 = zip(*model_data_sorted)\n",
    "\n",
    "# Assign the sorted data to the table\n",
    "for i in range(len(sorted_names_models)):\n",
    "    data[i][0] = sorted_names_models[i]\n",
    "    data[i][1] = sorted_level_00_acc[i]\n",
    "    data[i][2] = sorted_level_00_pre[i] \n",
    "    data[i][3] = sorted_level_00_rec[i] \n",
    "    data[i][4] = sorted_level_00_f1[i]\n",
    "\n",
    "# Define column headers\n",
    "headers = [\"Models\", \"ACC-00\", \"PRE-00\", \"REC-00\", \"F1-00\"]\n",
    "\n",
    "# Print the table\n",
    "table = tabulate(data, headers=headers, tablefmt=\"grid\")\n",
    "with open(output_file_name, \"a\") as f: print('Summary table - LEVEL 00', file = f)\n",
    "\n",
    "if feature_selection_bit == 1: \n",
    "    with open(output_file_name, \"a\") as f: print('Feature Selection was applied', file = f)\n",
    "else:\n",
    "    with open(output_file_name, \"a\") as f: print('All features were used', file = f)\n",
    "\n",
    "\n",
    "    \n",
    "print(table)\n",
    "with open(output_file_name, \"a\") as f: print(table, file = f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+----------------+\n",
      "| Models   |   time-00(sec) |\n",
      "+==========+================+\n",
      "| RF       |       0.479882 |\n",
      "+----------+----------------+\n",
      "| LGBM     |       3.2246   |\n",
      "+----------+----------------+\n",
      "| DT       |       3.5244   |\n",
      "+----------+----------------+\n",
      "| CAT      |       4.97263  |\n",
      "+----------+----------------+\n",
      "| Bag_rf   |       6.57582  |\n",
      "+----------+----------------+\n",
      "| LR       |       7.69426  |\n",
      "+----------+----------------+\n",
      "| SVM      |      16.1076   |\n",
      "+----------+----------------+\n",
      "| ADA      |      16.7214   |\n",
      "+----------+----------------+\n",
      "| XGB      |      16.7355   |\n",
      "+----------+----------------+\n",
      "| Bag_DT   |      23.0467   |\n",
      "+----------+----------------+\n",
      "| Bag_svm  |      27.0611   |\n",
      "+----------+----------------+\n",
      "| Bag_cat  |      51.1164   |\n",
      "+----------+----------------+\n",
      "| DNN      |     105.082    |\n",
      "+----------+----------------+\n",
      "| Bag_LR   |     113.124    |\n",
      "+----------+----------------+\n",
      "| KNN      |     113.696    |\n",
      "+----------+----------------+\n",
      "| Bag_ada  |     123.493    |\n",
      "+----------+----------------+\n",
      "| MLP      |     283.653    |\n",
      "+----------+----------------+\n",
      "| Bag_knn  |     666.486    |\n",
      "+----------+----------------+\n",
      "| Bag_comb |    1597.22     |\n",
      "+----------+----------------+\n",
      "| Bag_mlp  |    2059.45     |\n",
      "+----------+----------------+\n",
      "| Bag_lgbm |    6824.28     |\n",
      "+----------+----------------+\n"
     ]
    }
   ],
   "source": [
    "# implement time table\n",
    "from tabulate import tabulate\n",
    "\n",
    "names_models = ['ADA',\n",
    "                'SVM',\n",
    "                'DNN',\n",
    "                'MLP',\n",
    "                'KNN',\n",
    "                'CAT',\n",
    "                'XGB',\n",
    "                'LGBM',\n",
    "                'RF',\n",
    "                'LR',\n",
    "                'DT',\n",
    "                # 'VOTING',\n",
    "                'Bag_svm',\n",
    "                'Bag_knn',\n",
    "                'Bag_DT',\n",
    "                'Bag_LR',\n",
    "                'Bag_mlp',\n",
    "\n",
    "                'Bag_rf',\n",
    "                'Bag_ada',\n",
    "                'Bag_lgbm',\n",
    "                # 'Bag_xgb',\n",
    "                'Bag_cat',\n",
    "                'Bag_comb',\n",
    "                # 'avg',\n",
    "                # 'weighed_avg'\n",
    "                ]\n",
    "\n",
    "data = [[\"\" for _ in range(2)] for _ in range(len(names_models))]\n",
    "\n",
    "level_00_time = [\n",
    "                ada_time_00,\n",
    "                svm_time_00,\n",
    "                dnn_time_00,\n",
    "                mlp_time_00,\n",
    "                knn_time_00,\n",
    "                cat_time_00,\n",
    "                xgb_time_00,\n",
    "                lgbm_time_00,\n",
    "                rf_time_00,\n",
    "                lr_time_00,\n",
    "                dt_time_00,\n",
    "                # voting_time_00,\n",
    "                bag_svm_time_00,\n",
    "                bag_knn_time_00,\n",
    "                bag_dt_time_00,\n",
    "                bag_lr_time_00,\n",
    "                bag_mlp_time_00,\n",
    "\n",
    "                bag_rf_time_00,\n",
    "                bag_ada_time_00,\n",
    "                bag_lgbm_time_00,\n",
    "                # bag_xgb_time_00,\n",
    "                bag_cat_time_00,\n",
    "                bag_comb_time_00,\n",
    "\n",
    "                # avg_time_00,\n",
    "                # weighed_avg_time_00\n",
    "                ]  \n",
    "\n",
    "\n",
    "# Combine data into a list of tuples for sorting\n",
    "model_data = list(zip(names_models, level_00_time))\n",
    "\n",
    "# Sort by F1-00 score in descending order\n",
    "model_data_sorted = sorted(model_data, key=lambda x: x[1], reverse=False)\n",
    "\n",
    "# Separate the sorted data back into individual lists\n",
    "sorted_names_models, sorted_level_00_time = zip(*model_data_sorted)\n",
    "\n",
    "# Assign the sorted data to the table\n",
    "for i in range(len(sorted_names_models)):\n",
    "    data[i][0] = sorted_names_models[i]\n",
    "    data[i][1] = sorted_level_00_time[i]\n",
    "\n",
    "# Define column headers\n",
    "headers = [\"Models\", \"time-00(sec)\"]\n",
    "\n",
    "\n",
    "# Print the table\n",
    "table = tabulate(data, headers=headers, tablefmt=\"grid\")\n",
    "with open(output_file_name, \"a\") as f: print('Time is counted is seconds', file = f)\n",
    "print(table)\n",
    "with open(output_file_name, \"a\") as f: print(table, file = f)\n",
    "end_program = time.time()\n",
    "time_program = end_program - start_program\n",
    "with open(output_file_name, \"a\") as f: print('Running time of entire program is:', time_program ,' seconds',file = f)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.6.15 ('HITL': conda)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.15"
  },
  "vscode": {
   "interpreter": {
    "hash": "c4fae9402861b04134511d9dc79f354d2f3b6fe67518e49507b0a2b7cbc7bed8"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
